Refactoring queries and new logic about live markets info

live_data/markets_live_data.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:797e30e861c7468a5cf840a23b560d66bb9a1103d5bdae2cd86a485a43592e6e
+size 21197

live_data/markets_live_data_old.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:362255753caee81b4453f2d9c0fcf5870c770c3cf32d09c15ae6a571692a6874
+size 19052

scripts/live_markets_data.py

@@ -0,0 +1,276 @@
+#   -*- coding: utf-8 -*-
+#   ------------------------------------------------------------------------------
+#
+#     Copyright 2024 Valory AG
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+#
+#   ------------------------------------------------------------------------------
+
+import functools
+import warnings
+from typing import Optional, Generator, Callable
+import os
+import pandas as pd
+from datetime import datetime, timedelta, UTC
+import requests
+from tqdm import tqdm
+from typing import List, Dict
+from live_traders_data import add_trading_info
+from utils import SUBGRAPH_API_KEY, measure_execution_time
+from live_utils import OMEN_SUBGRAPH_URL, CREATOR, BATCH_SIZE, DATA_DIR
+from queries import (
+    FPMMS_WITH_TOKENS_QUERY,
+    ID_FIELD,
+    DATA_FIELD,
+    ANSWER_FIELD,
+    ANSWER_TIMESTAMP_FIELD,
+    QUERY_FIELD,
+    TITLE_FIELD,
+    OUTCOMES_FIELD,
+    OPENING_TIMESTAMP_FIELD,
+    CREATION_TIMESTAMP_FIELD,
+    LIQUIDITY_FIELD,
+    LIQUIDIY_MEASURE_FIELD,
+    TOKEN_AMOUNTS_FIELD,
+    ERROR_FIELD,
+    QUESTION_FIELD,
+    FPMMS_FIELD,
+)
+
+
+ResponseItemType = List[Dict[str, str]]
+SubgraphResponseType = Dict[str, ResponseItemType]
+
+
+class RetriesExceeded(Exception):
+    """Exception to raise when retries are exceeded during data-fetching."""
+
+    def __init__(
+        self, msg="Maximum retries were exceeded while trying to fetch the data!"
+    ):
+        super().__init__(msg)
+
+
+def hacky_retry(func: Callable, n_retries: int = 3) -> Callable:
+    """Create a hacky retry strategy.
+        Unfortunately, we cannot use `requests.packages.urllib3.util.retry.Retry`,
+        because the subgraph does not return the appropriate status codes in case of failure.
+        Instead, it always returns code 200. Thus, we raise exceptions manually inside `make_request`,
+        catch those exceptions in the hacky retry decorator and try again.
+        Finally, if the allowed number of retries is exceeded, we raise a custom `RetriesExceeded` exception.
+
+    :param func: the input request function.
+    :param n_retries: the maximum allowed number of retries.
+    :return: The request method with the hacky retry strategy applied.
+    """
+
+    @functools.wraps(func)
+    def wrapper_hacky_retry(*args, **kwargs) -> SubgraphResponseType:
+        """The wrapper for the hacky retry.
+
+        :return: a response dictionary.
+        """
+        retried = 0
+
+        while retried <= n_retries:
+            try:
+                if retried > 0:
+                    warnings.warn(f"Retrying {retried}/{n_retries}...")
+
+                return func(*args, **kwargs)
+            except (ValueError, ConnectionError) as e:
+                warnings.warn(e.args[0])
+            finally:
+                retried += 1
+
+        raise RetriesExceeded()
+
+    return wrapper_hacky_retry
+
+
+@hacky_retry
+def query_subgraph(url: str, query: str, key: str) -> SubgraphResponseType:
+    """Query a subgraph.
+
+    Args:
+        url: the subgraph's URL.
+        query: the query to be used.
+        key: the key to use in order to access the required data.
+
+    Returns:
+        a response dictionary.
+    """
+    content = {QUERY_FIELD: query}
+    headers = {
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+    }
+    res = requests.post(url, json=content, headers=headers)
+
+    if res.status_code != 200:
+        raise ConnectionError(
+            "Something went wrong while trying to communicate with the subgraph "
+            f"(Error: {res.status_code})!\n{res.text}"
+        )
+
+    body = res.json()
+    if ERROR_FIELD in body.keys():
+        raise ValueError(f"The given query is not correct: {body[ERROR_FIELD]}")
+
+    data = body.get(DATA_FIELD, {}).get(key, None)
+    if data is None:
+        raise ValueError(f"Unknown error encountered!\nRaw response: \n{body}")
+
+    return data
+
+
+def fpmms_fetcher(current_timestamp: int) -> Generator[ResponseItemType, int, None]:
+    """An indefinite fetcher for the FPMMs."""
+    omen_subgraph = OMEN_SUBGRAPH_URL.substitute(subgraph_api_key=SUBGRAPH_API_KEY)
+    print(f"omen_subgraph = {omen_subgraph}")
+    while True:
+        fpmm_id = yield
+        fpmms_query = FPMMS_WITH_TOKENS_QUERY.substitute(
+            creator=CREATOR,
+            fpmm_id=fpmm_id,
+            current_timestamp=current_timestamp,
+            fpmms_field=FPMMS_FIELD,
+            first=BATCH_SIZE,
+            id_field=ID_FIELD,
+            answer_timestamp_field=ANSWER_TIMESTAMP_FIELD,
+            question_field=QUESTION_FIELD,
+            outcomes_field=OUTCOMES_FIELD,
+            title_field=TITLE_FIELD,
+            opening_timestamp_field=OPENING_TIMESTAMP_FIELD,
+            creation_timestamp_field=CREATION_TIMESTAMP_FIELD,
+            liquidity_field=LIQUIDITY_FIELD,
+            liquidity_measure_field=LIQUIDIY_MEASURE_FIELD,
+            token_amounts_field=TOKEN_AMOUNTS_FIELD,
+        )
+        print(f"Executing query {fpmms_query}")
+        yield query_subgraph(omen_subgraph, fpmms_query, FPMMS_FIELD)
+
+
+def fetch_fpmms(current_timestamp: int) -> pd.DataFrame:
+    """Fetch all the fpmms of the creator."""
+    print("Fetching all markets")
+    latest_id = ""
+    fpmms = []
+    fetcher = fpmms_fetcher(current_timestamp)
+    for _ in tqdm(fetcher, unit="fpmms", unit_scale=BATCH_SIZE):
+        batch = fetcher.send(latest_id)
+        if len(batch) == 0:
+            print("no data")
+            break
+
+        # TODO Add the incremental batching system from market creator
+        # prev_fpmms is the previous local file with the markets
+        # for fpmm in batch:
+        #     if fpmm["id"] not in fpmms or "trades" not in prev_fpmms[fpmm["id"]]:
+        #         prev_fpmms[fpmm["id"]] = fpmm
+        print(f"length of the data received = {len(batch)}")
+        latest_id = batch[-1].get(ID_FIELD, "")
+        if latest_id == "":
+            raise ValueError(f"Unexpected data format retrieved: {batch}")
+
+        fpmms.extend(batch)
+
+    print("Finished collecting data")
+    return pd.DataFrame(fpmms)
+
+
+def get_answer(fpmm: pd.Series) -> str:
+    """Get an answer from its index, using Series of an FPMM."""
+    return fpmm[QUESTION_FIELD][OUTCOMES_FIELD][fpmm[ANSWER_FIELD]]
+
+
+def get_first_token_perc(row):
+    if row["total_tokens"] == 0.0:
+        return 0
+    return round((row["token_first_amount"] / row["total_tokens"]) * 100, 2)
+
+
+def get_second_token_perc(row):
+    if row["total_tokens"] == 0.0:
+        return 0
+    return round((row["token_second_amount"] / row["total_tokens"]) * 100, 2)
+
+
+def transform_fpmms(fpmms: pd.DataFrame, filename: str, current_timestamp: int) -> None:
+    """Transform an FPMMS dataframe."""
+
+    # prepare the new ones
+    # Add current timestamp
+    fpmms["tokens_timestamp"] = current_timestamp
+    fpmms["open"] = True
+
+    # computation of token distributions
+    fpmms["token_first_amount"] = fpmms.outcomeTokenAmounts.apply(lambda x: int(x[0]))
+    fpmms["token_second_amount"] = fpmms.outcomeTokenAmounts.apply(lambda x: int(x[1]))
+    fpmms["total_tokens"] = fpmms.apply(
+        lambda x: x.token_first_amount + x.token_second_amount, axis=1
+    )
+    fpmms["first_token_perc"] = fpmms.apply(lambda x: get_first_token_perc(x), axis=1)
+    fpmms["second_token_perc"] = fpmms.apply(lambda x: get_second_token_perc(x), axis=1)
+    fpmms.drop(
+        columns=["token_first_amount", "token_second_amount", "total_tokens"],
+        inplace=True,
+    )
+    # previous file to update?
+    old_fpmms = None
+    if os.path.exists(DATA_DIR / filename):
+        old_fpmms = pd.read_parquet(DATA_DIR / filename)
+
+    if old_fpmms is not None:
+        # update which markets are not open anymore
+        open_markets = list(fpmms.id.unique())
+        print("Updating market status of old markets")
+        open_mask = old_fpmms["id"].isin(open_markets)
+        old_fpmms.loc[~open_mask, "status"] = False
+
+        # now concatenate
+        print("Appending new data to previous data")
+        fpmms = pd.concat([old_fpmms, fpmms], ignore_index=True)
+        # fpmms.drop_duplicates(inplace=True)
+
+    return
+
+
+@measure_execution_time
+def compute_distributions(filename: Optional[str]) -> pd.DataFrame:
+    """Fetch, process, store and return the markets as a Dataframe."""
+
+    print("fetching new markets information")
+    current_timestamp = int(datetime.now(UTC).timestamp())
+    fpmms = fetch_fpmms(current_timestamp)
+    print(fpmms.head())
+
+    print("transforming and updating previous data")
+
+    transform_fpmms(fpmms, filename, current_timestamp)
+    print(fpmms.head())
+
+    # WIP
+    # print("Adding trading information")
+    add_trading_info(fpmms)
+    print("saving the data")
+    print(fpmms.info())
+    if filename:
+        fpmms.to_parquet(DATA_DIR / filename, index=False)
+
+    return fpmms
+
+
+if __name__ == "__main__":
+    compute_distributions("markets_live_data.parquet")

scripts/live_traders_data.py

@@ -0,0 +1,153 @@
+import requests
+import os
+from datetime import datetime, timedelta, UTC
+import pandas as pd
+from collections import defaultdict
+from typing import Any, Optional
+from tqdm import tqdm
+from live_utils import OMEN_SUBGRAPH_URL, CREATOR, BATCH_SIZE, DATA_DIR
+from utils import SUBGRAPH_API_KEY, _to_content
+from queries import omen_trader_votes_query
+
+
+headers = {
+    "Accept": "application/json, multipart/mixed",
+    "Content-Type": "application/json",
+}
+
+
+def _query_omen_xdai_subgraph(
+    fpmm_id: str,
+) -> dict[str, Any]:
+    """Query the subgraph."""
+    omen_subgraph = OMEN_SUBGRAPH_URL.substitute(subgraph_api_key=SUBGRAPH_API_KEY)
+    print(f"omen_subgraph = {omen_subgraph}")
+    grouped_results = defaultdict(list)
+    id_gt = ""
+
+    while True:
+        query = omen_trader_votes_query.substitute(
+            fpmm_creator=CREATOR.lower(),
+            first=BATCH_SIZE,
+            id_gt=id_gt,
+            fpmm_id=fpmm_id,
+        )
+        print(f"query for the omen to collect trades {query}")
+        content_json = _to_content(query)
+
+        res = requests.post(omen_subgraph, headers=headers, json=content_json)
+        result_json = res.json()
+        # print(f"result = {result_json}")
+        user_trades = result_json.get("data", {}).get("fpmmTrades", [])
+
+        if not user_trades:
+            break
+
+        for trade in user_trades:
+            fpmm_id = trade.get("fpmm", {}).get("id")
+            grouped_results[fpmm_id].append(trade)
+
+        id_gt = user_trades[len(user_trades) - 1]["id"]
+
+    all_results = {
+        "data": {
+            "fpmmTrades": [
+                trade
+                for trades_list in grouped_results.values()
+                for trade in trades_list
+            ]
+        }
+    }
+
+    return all_results
+
+
+def transform_trades(trades_json: dict) -> pd.DataFrame:
+    # convert to dataframe
+    print("transforming trades")
+    df = pd.DataFrame(trades_json["data"]["fpmmTrades"])
+    if len(df) == 0:
+        print("No trades for this market")
+        return df
+
+    # print(df.info())
+
+    # convert creator to address
+    df["trade_creator"] = df["creator"].apply(lambda x: x["id"])
+
+    # normalize fpmm column
+    fpmm = pd.json_normalize(df["fpmm"])
+    fpmm.columns = [f"fpmm.{col}" for col in fpmm.columns]
+    df = pd.concat([df, fpmm], axis=1)
+
+    # drop fpmm column
+    df.drop(["fpmm"], axis=1, inplace=True)
+
+    # convert into int
+    df.outcomeIndex = pd.to_numeric(df.outcomeIndex, errors="coerce")
+    return df
+
+
+def compute_from_timestamp_value(
+    fpmm_id: str, opening_timestamp: int, fpmms: pd.DataFrame
+) -> Optional[int]:
+    """Function to find the latest timestamp registered for a specific market"""
+    try:
+        market_data = fpmms.loc[fpmms["id"] == fpmm_id]
+        # how many previous samples do we have?
+        if len(market_data) == 1:
+            # take the opening Timestamp of the Market
+            return opening_timestamp
+        timestamps = (market_data.tokens_timestamp.values).sort()
+        # the last value is the current timestamp so we need to take the previous one
+        return timestamps[-2]
+    except Exception as e:
+        print(
+            f"Error when trying to get the from timestamp value of the market id {fpmm_id}"
+        )
+        return None
+
+
+def compute_votes_distribution(market_trades: pd.DataFrame):
+    """Function to compute the distribution of votes for the trades of a market"""
+    total_trades = len(market_trades)
+    print(f"The total number of trades is {total_trades}")
+    # outcomeIndex is always 1 or 0?
+    sum_outcome_index_1 = sum(market_trades.outcomeIndex)
+    print(f"The total number of votes for index 1 is {sum_outcome_index_1}")
+    percentage_index_1 = round((sum_outcome_index_1 / total_trades) * 100, 2)
+    return (100 - percentage_index_1), percentage_index_1
+
+
+def add_trading_info(fpmms: pd.DataFrame) -> None:
+    # Iterate over the markets
+    print("Adding votes distribution per market")
+    fpmms["votes_first_outcome_perc"] = 0.0
+    fpmms["votes_second_outcome_perc"] = 0.0
+    for i, fpmm in tqdm(fpmms.iterrows(), total=len(fpmms), desc="Analysing trades"):
+        # read trades from latest read timestamp
+        market_id = fpmm["id"]
+        print(f"Adding information for the market {market_id}")
+        market_trades_json = _query_omen_xdai_subgraph(
+            fpmm_id=market_id,
+        )
+        market_trades = transform_trades(market_trades_json)
+        if len(market_trades) == 0:
+            continue
+        # to compute the votes distribution
+        print("Computing the votes distribution")
+        first_outcome, second_outcome = compute_votes_distribution(market_trades)
+        print(
+            f"first outcome votes ={first_outcome}, second outcome votes = {second_outcome}"
+        )
+        fpmms.loc[fpmms["id"] == market_id, "votes_first_outcome_perc"] = first_outcome
+        fpmms.loc[fpmms["id"] == market_id, "votes_second_outcome_perc"] = (
+            second_outcome
+        )
+    print("Dataset after adding trading info")
+    print(fpmms.head())
+    return
+
+
+if __name__ == "__main__":
+    print("collecting votes distribution")

scripts/live_utils.py

@@ -0,0 +1,14 @@
+from string import Template
+from pathlib import Path
+
+
+CREATOR = "0x89c5cc945dd550BcFfb72Fe42BfF002429F46Fec"
+BATCH_SIZE = 1000
+# OMEN_SUBGRAPH = "https://api.thegraph.com/subgraphs/name/protofire/omen-xdai"
+OMEN_SUBGRAPH_URL = Template(
+    """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/9fUVQpFwzpdWS9bq5WkAnmKbNNcoBwatMR4yZq81pbbz"""
+)
+SCRIPTS_DIR = Path(__file__).parent
+ROOT_DIR = SCRIPTS_DIR.parent
+DATA_DIR = ROOT_DIR / "live_data"
+MAX_UINT_HEX = "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"

scripts/markets.py

@@ -29,6 +29,18 @@ from tqdm import tqdm
 from typing import List, Dict
 from pathlib import Path
 from utils import SUBGRAPH_API_KEY
+from queries import (
+    FPMMS_QUERY,
+    ID_FIELD,
+    DATA_FIELD,
+    ANSWER_FIELD,
+    QUERY_FIELD,
+    TITLE_FIELD,
+    OUTCOMES_FIELD,
+    ERROR_FIELD,
+    QUESTION_FIELD,
+    FPMMS_FIELD,
+)
 
 ResponseItemType = List[Dict[str, str]]
 SubgraphResponseType = Dict[str, ResponseItemType]
@@ -40,42 +52,12 @@ BATCH_SIZE = 1000
 OMEN_SUBGRAPH_URL = Template(
     """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/9fUVQpFwzpdWS9bq5WkAnmKbNNcoBwatMR4yZq81pbbz"""
 )
-FPMMS_FIELD = "fixedProductMarketMakers"
-QUERY_FIELD = "query"
-ERROR_FIELD = "errors"
-DATA_FIELD = "data"
-ID_FIELD = "id"
-ANSWER_FIELD = "currentAnswer"
-QUESTION_FIELD = "question"
-OUTCOMES_FIELD = "outcomes"
-TITLE_FIELD = "title"
+
 MAX_UINT_HEX = "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
 DEFAULT_FILENAME = "fpmms.parquet"
 SCRIPTS_DIR = Path(__file__).parent
 ROOT_DIR = SCRIPTS_DIR.parent
 DATA_DIR = ROOT_DIR / "data"
-FPMMS_QUERY = Template(
-    """
-    {
-      ${fpmms_field}(
-        where: {
-          creator: "${creator}",
-          id_gt: "${fpmm_id}",
-          isPendingArbitration: false
-        },
-        orderBy: ${id_field}
-        first: ${first}
-      ){
-        ${id_field}
-        ${answer_field}
-        ${question_field} {
-          ${outcomes_field}
-        }
-        ${title_field}
-      }
-    }
-    """
-)
 
 
 class RetriesExceeded(Exception):

scripts/profitability.py

@@ -30,20 +30,9 @@ import numpy as np
 import os
 from pathlib import Path
 from get_mech_info import DATETIME_60_DAYS_AGO
-from utils import SUBGRAPH_API_KEY, wei_to_unit, convert_hex_to_int
-
-IRRELEVANT_TOOLS = [
-    "openai-text-davinci-002",
-    "openai-text-davinci-003",
-    "openai-gpt-3.5-turbo",
-    "openai-gpt-4",
-    "stabilityai-stable-diffusion-v1-5",
-    "stabilityai-stable-diffusion-xl-beta-v2-2-2",
-    "stabilityai-stable-diffusion-512-v2-1",
-    "stabilityai-stable-diffusion-768-v2-1",
-    "deepmind-optimization-strong",
-    "deepmind-optimization",
-]
+from utils import SUBGRAPH_API_KEY, wei_to_unit, convert_hex_to_int, _to_content
+from queries import omen_xdai_trades_query, conditional_tokens_gc_user_query
+
 QUERY_BATCH_SIZE = 1000
 DUST_THRESHOLD = 10000000000000
 INVALID_ANSWER = -1
@@ -52,7 +41,7 @@ DEFAULT_FROM_DATE = "1970-01-01T00:00:00"
 DEFAULT_TO_DATE = "2038-01-19T03:14:07"
 DEFAULT_FROM_TIMESTAMP = 0
 DEFAULT_60_DAYS_AGO_TIMESTAMP = (DATETIME_60_DAYS_AGO).timestamp()
-DEFAULT_TO_TIMESTAMP = 2147483647
+DEFAULT_TO_TIMESTAMP = 2147483647  # around year 2038
 WXDAI_CONTRACT_ADDRESS = "0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d"
 DEFAULT_MECH_FEE = 0.01
 DUST_THRESHOLD = 10000000000000
@@ -153,96 +142,6 @@ headers = {
 }
 
 
-omen_xdai_trades_query = Template(
-    """
-    {
-        fpmmTrades(
-            where: {
-                type: Buy,
-                fpmm_: {
-                    creator: "${fpmm_creator}"
-                    creationTimestamp_gte: "${fpmm_creationTimestamp_gte}",
-                    creationTimestamp_lt: "${fpmm_creationTimestamp_lte}"
-                },
-                creationTimestamp_gte: "${creationTimestamp_gte}",
-                creationTimestamp_lte: "${creationTimestamp_lte}"
-                id_gt: "${id_gt}"
-            }
-            first: ${first}
-            orderBy: id
-            orderDirection: asc
-        ) {
-            id
-            title
-            collateralToken
-            outcomeTokenMarginalPrice
-            oldOutcomeTokenMarginalPrice
-            type
-            creator {
-                id
-            }
-            creationTimestamp
-            collateralAmount
-            collateralAmountUSD
-            feeAmount
-            outcomeIndex
-            outcomeTokensTraded
-            transactionHash
-            fpmm {
-                id
-                outcomes
-                title
-                answerFinalizedTimestamp
-                currentAnswer
-                isPendingArbitration
-                arbitrationOccurred
-                openingTimestamp
-                condition {
-                    id
-                }
-            }
-        }
-    }
-    """
-)
-
-
-conditional_tokens_gc_user_query = Template(
-    """
-    {
-        user(id: "${id}") {
-            userPositions(
-                first: ${first}
-                where: {
-                    id_gt: "${userPositions_id_gt}"
-                }
-                orderBy: id
-            ) {
-                balance
-                id
-                position {
-                    id
-                    conditionIds
-                }
-                totalBalance
-                wrappedBalance
-            }
-        }
-    }
-    """
-)
-
-
-def _to_content(q: str) -> dict[str, Any]:
-    """Convert the given query string to payload content, i.e., add it under a `queries` key and convert it to bytes."""
-    finalized_query = {
-        "query": q,
-        "variables": None,
-        "extensions": {"headers": None},
-    }
-    return finalized_query
-
-
 def _query_omen_xdai_subgraph(
     from_timestamp: float,
     to_timestamp: float,

scripts/queries.py

@@ -0,0 +1,240 @@
+#   -*- coding: utf-8 -*-
+#   ------------------------------------------------------------------------------
+#
+#     Copyright 2024 Valory AG
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+#
+#   ------------------------------------------------------------------------------
+
+from string import Template
+
+FPMMS_FIELD = "fixedProductMarketMakers"
+QUERY_FIELD = "query"
+ERROR_FIELD = "errors"
+DATA_FIELD = "data"
+ID_FIELD = "id"
+ANSWER_FIELD = "currentAnswer"
+QUESTION_FIELD = "question"
+OUTCOMES_FIELD = "outcomes"
+TITLE_FIELD = "title"
+ANSWER_TIMESTAMP_FIELD = "currentAnswerTimestamp"
+OPENING_TIMESTAMP_FIELD = "openingTimestamp"
+RESOLUTION_TIMESTAMP_FIELD = "resolutionTimestamp"
+CREATION_TIMESTAMP_FIELD = "creationTimestamp"
+LIQUIDITY_FIELD = "liquidityParameter"
+LIQUIDIY_MEASURE_FIELD = "liquidityMeasure"
+TOKEN_AMOUNTS_FIELD = "outcomeTokenAmounts"
+
+FPMMS_QUERY = Template(
+    """
+    {
+      ${fpmms_field}(
+        where: {
+          creator: "${creator}",
+          id_gt: "${fpmm_id}",
+          isPendingArbitration: false
+        },
+        orderBy: ${id_field}
+        first: ${first}
+      ){
+        ${id_field}
+        ${answer_field}
+        ${question_field} {
+          ${outcomes_field}
+        }
+        ${title_field}
+      }
+    }
+    """
+)
+
+FPMMS_WITH_TOKENS_QUERY = Template(
+    """
+    {
+      ${fpmms_field}(
+        where: {
+          creator: "${creator}",
+          id_gt: "${fpmm_id}",
+          isPendingArbitration: false
+          currentAnswer: null
+          openingTimestamp_gt:${current_timestamp}
+        },
+        orderBy: ${id_field}
+        orderDirection: asc
+        first: ${first}
+      ){
+        ${id_field}
+        ${question_field} {
+          ${outcomes_field}
+          ${answer_timestamp_field}
+          answers{
+          answer
+          }
+        }
+        ${title_field}
+        ${opening_timestamp_field}
+        ${creation_timestamp_field}
+        ${liquidity_field}
+        ${liquidity_measure_field}
+        ${token_amounts_field}
+      }
+    }
+    """
+)
+omen_xdai_trades_query = Template(
+    """
+    {
+        fpmmTrades(
+            where: {
+                type: Buy,
+                fpmm_: {
+                    creator: "${fpmm_creator}"
+                    creationTimestamp_gte: "${fpmm_creationTimestamp_gte}",
+                    creationTimestamp_lt: "${fpmm_creationTimestamp_lte}"
+                },
+                creationTimestamp_gte: "${creationTimestamp_gte}",
+                creationTimestamp_lte: "${creationTimestamp_lte}"
+                id_gt: "${id_gt}"
+            }
+            first: ${first}
+            orderBy: id
+            orderDirection: asc
+        ) {
+            id
+            title
+            collateralToken
+            outcomeTokenMarginalPrice
+            oldOutcomeTokenMarginalPrice
+            type
+            creator {
+                id
+            }
+            creationTimestamp
+            collateralAmount
+            collateralAmountUSD
+            feeAmount
+            outcomeIndex
+            outcomeTokensTraded
+            transactionHash
+            fpmm {
+                id
+                outcomes
+                title
+                answerFinalizedTimestamp
+                currentAnswer
+                isPendingArbitration
+                arbitrationOccurred
+                openingTimestamp
+                condition {
+                    id
+                }
+            }
+        }
+    }
+    """
+)
+
+omen_trader_votes_query = Template(
+    """
+    {
+        fpmmTrades(
+            where: {
+                type: Buy,
+                fpmm_: {
+                    creator: "${fpmm_creator}",
+                    id: "${fpmm_id}",
+                },
+                id_gt: "${id_gt}"
+            }
+            first: ${first}
+            orderBy: id
+            orderDirection: asc
+        ) {
+            id
+            title
+            collateralToken
+            outcomeTokenMarginalPrice
+            oldOutcomeTokenMarginalPrice
+            type
+            creator {
+                id
+            }
+            creationTimestamp
+            collateralAmount
+            collateralAmountUSD
+            feeAmount
+            outcomeIndex
+            outcomeTokensTraded
+            transactionHash
+            fpmm {
+                id
+                outcomes
+                title
+                condition {
+                    id
+                }
+            }
+        }
+    }
+    """
+)
+
+
+conditional_tokens_gc_user_query = Template(
+    """
+    {
+        user(id: "${id}") {
+            userPositions(
+                first: ${first}
+                where: {
+                    id_gt: "${userPositions_id_gt}"
+                }
+                orderBy: id
+            ) {
+                balance
+                id
+                position {
+                    id
+                    conditionIds
+                }
+                totalBalance
+                wrappedBalance
+            }
+        }
+    }
+    """
+)
+
+
+TRADES_QUERY = """
+query fpmms_query($fpmm: String, $id_gt: ID) {
+    fpmmTrades(
+        where: {fpmm: $fpmm, id_gt: $id_gt, type: Buy}
+        orderBy: id
+        orderDirection: asc
+        first: 1000
+    ) {
+        collateralAmount
+        outcomeIndex
+        outcomeTokensTraded
+        id
+        oldOutcomeTokenMarginalPrice
+        outcomeTokenMarginalPrice
+        type
+        collateralAmountUSD
+        creationTimestamp
+        feeAmount
+  }
+}
+"""

scripts/utils.py

@@ -4,6 +4,7 @@ import os
 import time
 from tqdm import tqdm
 from typing import List, Any, Optional, Union
+import numpy as np
 import pandas as pd
 import gc
 import re
@@ -348,3 +349,25 @@ def convert_hex_to_int(x: Union[str, float]) -> Union[int, float]:
 def wei_to_unit(wei: int) -> float:
     """Converts wei to currency unit."""
     return wei / 10**18
+
+
+def measure_execution_time(func):
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = func(*args, **kwargs)
+        end_time = time.time()
+        execution_time = end_time - start_time
+        print(f"Execution time: {execution_time:.6f} seconds")
+        return result
+
+    return wrapper
+
+
+def _to_content(q: str) -> dict[str, Any]:
+    """Convert the given query string to payload content, i.e., add it under a `queries` key and convert it to bytes."""
+    finalized_query = {
+        "query": q,
+        "variables": None,
+        "extensions": {"headers": None},
+    }
+    return finalized_query