Pagination and Timeframes

{
  "status": 200,
  "title": "OK",
  "description": "Successful request",
  "payload": {
    "metadata": {
      "next": "https://api.amberdata.com/api/v2/defi/lending/compoundv2/assets/ZRX?cursor=N4IgRgNg9gxg1jAFgQwJYDsCSATEAuEARhgE5SA2EgBnIFYAzGbY%2BkAGhAAcAnKAF1hQIOfCBhQAtpygBXdNgBuAJnYgJybnACmffOhkQIHDDAgzsWgIKGAKt2ToAzshh9UUdAFEFW9H0f49MgQjlocbhJaAGJQ3Oq6BIgy6ugASlrI2MiQWqrYqNxaru7oopYAygDCqo58GnwAIsh8uXiE5OSEACwkXYQA7AAcVCMcvthNLfjtnSTUXSOLHI6oAF6thKMgFj7QnFrcALJQFoHBoRz0sfGiAFaOHqqcyADmGxyxqC8YweV13I1mhs6CQAMyEJSLJYgT7fdDBTzySbA2bzKFUDjIRyhPgiAhUAAeWj62CUXUGJFo-UI2FBSiUhApZPIWSCtC0YAshHoXUoJB5JEGqggWL4NlQkVqyCkohm3V6AyFHBFtQAMlAXph5FoCdNwQBfIA"
    },
    ...
  }
}

import os
from datetime import datetime
from dataclasses import dataclass, field

from endpoint_timerange_handler import EndpointTimeRangeHandler, Endpoint
from endpoint_caller_v2 import EndpointCaller

def http_ok_next_page_url_extractor(page):
    """
    Function that extracts the next page url from the current page of data

    Parameters
    ----------
    page: dict

    Returns
    -------
    next_page_url: str
    """
    if 'payload' in page and page['payload'] is not None:
        payload = page['payload']
        if 'metadata' in payload and payload['metadata'] is not None:
            metadata = payload['metadata']
            if 'next' in metadata and metadata['next'] is not None:
                next_page_url = metadata['next']
                return next_page_url
    return ""

@dataclass(kw_only=True)
class DEXTradesHistorical(Endpoint):
    poolAddress: str
    path_template: str = field(default='/market/defi/trades/{}/historical')
    max_interval_in_seconds: int = field(default=2592000) #30 days * 24 hrs * 60 min * 60 seconds

    def format_path(self) -> str:
        return self.path_template.format(self.poolAddress)

def get_api_responses(start_date: datetime, end_date: datetime, endpoint: DEXTradesHistorical) -> None:
    """
    Get all pages of data between `start_date` and `end_date`.
    """
    endpoint_caller = EndpointCaller(os.getenv('PRODUCTION_API_KEY'))
    endpoint_timerange_handler = EndpointTimeRangeHandler(endpoint_caller)
    for page in endpoint_timerange_handler.get_data_for_timerange(
        start_date,
        end_date,
        endpoint,
        http_ok_next_page_url_extractor
    ):
        response = page.data
        print(f"Timestamp of first entry in the page: {response['payload']['data'][0][1]}")

def call_dex_trades_historical() -> None:
    """
    Example configuration of an endpoint to be called.
    """
    start_date = "2022-01-01T00:00:00"
    end_date = "2023-01-01T00:00:00"
    start_date_as_dt = datetime.fromisoformat(start_date).replace(microsecond=0)
    end_date_as_dt = datetime.fromisoformat(end_date).replace(microsecond=0)
    poolAddress = '0xcbcdf9626bc03e24f779434178a73a0b4bad62ed' # WBTC/ETH 0.3%
    exchange = 'uniswapv3'

    dex_trades_historical = DEXTradesHistorical(poolAddress=poolAddress)
    dex_trades_historical.add_query_parameter('exchange', exchange)
    get_api_responses(start_date_as_dt, end_date_as_dt, dex_trades_historical)

if __name__ == "__main__":
    call_dex_trades_historical()

from datetime import datetime, timedelta
from dataclasses import dataclass, field
from endpoint_caller_v2 import EndpointCaller

@dataclass(kw_only=True)
class Endpoint:
    """
    This is a parent class that should be inherited and implemented for specific endpoints.
    """
    query: dict = field(default_factory=dict)
    headers: dict = field(default_factory=dict)

    def add_query_parameter(self, parameter_name: str, parameter_value) -> None:
        if parameter_name is not None and len(parameter_name) > 0:
            self.query[parameter_name] = parameter_value

    def format_path(self) -> str:
        """
        Child classes must implement this function.
        """
        return ""

    def add_header(self, header_name: str, header_value: str) -> None:
        if header_name is not None and len(header_name) > 0:
            self.headers[header_name] = header_value

class EndpointTimeRangeHandler:
    def __init__(self, endpoint_caller: EndpointCaller) -> None:
        self.endpoint_caller = endpoint_caller

    def get_data_for_timerange(self, start_date: datetime, end_date: datetime,
                              endpoint: Endpoint, http_ok_next_page_url_extractor):
        """
        Given an arbitrarily large timerange, this function will call the endpoint
        continuously by breaking up the requested time range into chunks.

        The chunks do not exceed the single request maximum range (endDate - startDate)
        for the specific endpoint.
        """
        duration = end_date - start_date
        intervals = duration.total_seconds()/endpoint.max_interval_in_seconds
        hours = duration.total_seconds()/3600
        print(f"Getting {hours} hours of data (# of intervals: {intervals})")

        start_date_copy = start_date
        timerange_stack = []
        while start_date_copy + timedelta(seconds=endpoint.max_interval_in_seconds) <= end_date:
            intermediate_end_date = start_date_copy + timedelta(seconds=endpoint.max_interval_in_seconds)
            timerange_stack.append((start_date_copy, intermediate_end_date))
            start_date_copy = intermediate_end_date
        timerange_stack.append((start_date_copy, end_date))
        timerange_stack.reverse()

        while len(timerange_stack) > 0:
            timerange = timerange_stack.pop()
            print(f"Retrieving data from {str(timerange[0])} to {str(timerange[1])}")
            yield from self.get_data(timerange[0], timerange[1], endpoint, http_ok_next_page_url_extractor)