1d23b992191fde5403c1927efd339516fbee1a8c,QUANTAXIS/QASU/save_bitfinex.py,,QA_SU_save_bitfinex_min,#Any#Any#Any#,178

Before Change

    
    print("Under construction... I will test and debug soon...")
    return False
    market = "bitfinex"
    symbol_list = QA_fetch_crypto_asset_list(market=market)
    col = DATABASE.crypto_asset_min
    col.create_index(
        [
            ("market",
             pymongo.ASCENDING),
            ("symbol",
             pymongo.ASCENDING),
            ("time_stamp",
             pymongo.ASCENDING),
            ("date_stamp",
             pymongo.ASCENDING)
        ]
    )
    col.create_index(
        [
            ("market",
             pymongo.ASCENDING),
            ("symbol",
             pymongo.ASCENDING),
            ("type",
             pymongo.ASCENDING),
            ("time_stamp",
             pymongo.ASCENDING)
        ],
        unique=True
    )

    end = datetime.datetime.now(tzutc())

    QA_util_log_info(
        "Starting DOWNLOAD PROGRESS of min Klines from Bitfinex... ",
        ui_log=ui_log,
        ui_progress=ui_progress
    )
    for index in range(len(symbol_list)):
        symbol_info = symbol_list.iloc[index]
        // 上架仅处理交易对
        QA_util_log_info(
            "The "{}" //{} of total in {}".format(
                symbol_info["symbol"],
                index,
                len(symbol_list)
            ),
            ui_log=ui_log,
            ui_progress=ui_progress
        )
        QA_util_log_info(
            "DOWNLOAD PROGRESS {} "
            .format(str(float(index / len(symbol_list) * 100))[0:4] + "%"),
            ui_log=ui_log,
            ui_progress=ui_progress
        )
        query_id = {
            "symbol": symbol_info["symbol"],
            "market": symbol_info["market"],
            "type": Bitfinex2QA_FREQUENCY_DICT[frequency]
        }
        ref = col.find(query_id).sort("time_stamp", -1)

        if (col.count_documents(query_id) > 0):
            start_stamp = ref.next()["time_stamp"]
            start_time = datetime.datetime.fromtimestamp(
                start_stamp + 1,
                tz=tzutc()
            )
            QA_util_log_info(
                "UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}".format(
                    symbol_info["symbol"],
                    Bitfinex2QA_FREQUENCY_DICT[frequency],
                    QA_util_timestamp_to_str(start_time),
                    QA_util_timestamp_to_str(end)
                ),
                ui_log=ui_log,
                ui_progress=ui_progress
            )

            // 查询到 Kline 缺漏，点抓取模式，按缺失的时间段精确请求K线数据
            missing_data_list = QA_util_find_missing_kline(
                symbol_info["symbol"],
                OKEx2QA_FREQUENCY_DICT[frequency],
                market="okex"
            )[::-1]
        else:
            start_time = Bitfinex_MIN_DATE
            QA_util_log_info(
                "NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}".format(
                    symbol_info["symbol"],
                    Bitfinex2QA_FREQUENCY_DICT[frequency],
                    QA_util_timestamp_to_str(start_time),
                    QA_util_timestamp_to_str(end)
                ),
                ui_log=ui_log,
                ui_progress=ui_progress
            )
            miss_kline = pd.DataFrame(
                [
                    [
                        int(QA_util_datetime_to_Unix_timestamp(start_time)),
                        int(QA_util_datetime_to_Unix_timestamp(end)),
                        "{} to {}".format(start_time,
                                         end)
                    ]
                ],
                columns=["expected",
                         "between",
                         "missing"]
            )
            missing_data_list = miss_kline.values

        if len(missing_data_list) > 0:
            // 查询确定中断的K线数据起止时间，缺分时数据，补分时数据
            expected = 0
            between = 1
            missing = 2
            reqParams = {}
            for i in range(len(missing_data_list)):
                reqParams["from"] = int(missing_data_list[i][expected])
                reqParams["to"] = int(missing_data_list[i][between])
                if (reqParams["from"] >
                    (QA_util_datetime_to_Unix_timestamp() + 120)):
                    // 出现“未来”时间，一般是默认时区设置错误造成的
                    raise Exception(
                        "A unexpected \"Future\" timestamp got, Please check self.missing_data_list_func param \"tzlocalize\" set. More info: {:s}@{:s} at {:s} but current time is {}"
                        .format(
                            symbol_info["symbol"],
                            frequency,
                            QA_util_print_timestamp(reqParams["from"]),
                            QA_util_print_timestamp(
                                QA_util_datetime_to_Unix_timestamp()
                            )
                        )
                    )
                QA_util_log_info(
                    "Fetch "{:s}" slices "{:s}" kline：{:s} to {:s}".format(
                        symbol_info["symbol"],
                        OKEx2QA_FREQUENCY_DICT[frequency],
                        QA_util_timestamp_to_str(
                            missing_data_list[i][expected]

After Change

    
    Save Bitfinex min kline 分钟线数据，统一转化字段保存数据为 crypto_asset_min
    
    symbol_template = Bitfinex_SYMBOL
    symbol_list = QA_fetch_cryptocurrency_list(Bitfinex_EXCHANGE)
    col = DATABASE.cryptocurrency_min
    col.create_index(
        [
            ("symbol",
             pymongo.ASCENDING),
            ("time_stamp",
             pymongo.ASCENDING),
            ("date_stamp",
             pymongo.ASCENDING)
        ]
    )
    col.create_index(
        [
            ("symbol",
             pymongo.ASCENDING),
            ("type",
             pymongo.ASCENDING),
            ("time_stamp",
             pymongo.ASCENDING)
        ],
        unique=True
    )

    end = datetime.datetime.now(tzutc())

    QA_util_log_info(
        "Starting DOWNLOAD PROGRESS of min Klines from {:s}... ".format(Bitfinex_EXCHANGE),
        ui_log=ui_log,
        ui_progress=ui_progress
    )
    for index in range(len(symbol_list)):
        symbol_info = symbol_list.iloc[index]
        // 上架仅处理交易对
        QA_util_log_info(
            "The "{}" //{} of total in {}".format(
                symbol_template.format(symbol_info["symbol"]),
                index,
                len(symbol_list)
            ),
            ui_log=ui_log,
            ui_progress=ui_progress
        )
        QA_util_log_info(
            "DOWNLOAD PROGRESS {} "
            .format(str(float(index / len(symbol_list) * 100))[0:4] + "%"),
            ui_log=ui_log,
            ui_progress=ui_progress
        )
        query_id = {
            "symbol": symbol_template.format(symbol_info["symbol"]),
            "type": Bitfinex2QA_FREQUENCY_DICT[frequency]
        }
        ref = col.find(query_id).sort("time_stamp", -1)

        if (col.count_documents(query_id) > 0):
            start_stamp = ref.next()["time_stamp"]
            start_time = datetime.datetime.fromtimestamp(
                start_stamp + 1,
                tz=tzutc()
            )
            QA_util_log_info(
                "UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}".format(
                    symbol_template.format(symbol_info["symbol"]),
                    Bitfinex2QA_FREQUENCY_DICT[frequency],
                    QA_util_timestamp_to_str(start_time),
                    QA_util_timestamp_to_str(end)
                ),
                ui_log=ui_log,
                ui_progress=ui_progress
            )

            // 查询到 Kline 缺漏，点抓取模式，按缺失的时间段精确请求K线数据
            missing_data_list = QA_util_find_missing_kline(
                symbol_info["symbol"],
                Bitfinex2QA_FREQUENCY_DICT[frequency],
            )[::-1]
        else:
            start_time = Bitfinex_MIN_DATE
            QA_util_log_info(
                "NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}".format(
                    symbol_template.format(symbol_info["symbol"]),
                    Bitfinex2QA_FREQUENCY_DICT[frequency],
                    QA_util_timestamp_to_str(start_time),
                    QA_util_timestamp_to_str(end)
                ),
                ui_log=ui_log,
                ui_progress=ui_progress
            )
            miss_kline = pd.DataFrame(
                [
                    [
                        int(QA_util_datetime_to_Unix_timestamp(start_time)),
                        int(QA_util_datetime_to_Unix_timestamp(end)),
                        "{} to {}".format(start_time,
                                         end)
                    ]
                ],
                columns=["expected",
                         "between",
                         "missing"]
            )
            missing_data_list = miss_kline.values

        if len(missing_data_list) > 0:
            // 查询确定中断的K线数据起止时间，缺分时数据，补分时数据
            expected = 0
            between = 1
            missing = 2
            reqParams = {}
            for i in range(len(missing_data_list)):
                reqParams["from"] = int(missing_data_list[i][expected])
                reqParams["to"] = int(missing_data_list[i][between])
                if (reqParams["from"] >
                    (QA_util_datetime_to_Unix_timestamp() + 120)):
                    // 出现“未来”时间，一般是默认时区设置错误造成的
                    QA_util_log_info(
                        "A unexpected \"Future\" timestamp got, Please check self.missing_data_list_func param \"tzlocalize\" set. More info: {:s}@{:s} at {:s} but current time is {}"
                        .format(
                            symbol_template.format(symbol_info["symbol"]),
                            frequency,
                            QA_util_print_timestamp(reqParams["from"]),
                            QA_util_print_timestamp(
                                QA_util_datetime_to_Unix_timestamp()
                            )
                        )
                    )
                    // 跳到下一个时间段
                    continue

                QA_util_log_info(
                    "Fetch "{:s}" slices "{:s}" kline：{:s} to {:s}".format(
                        symbol_template.format(symbol_info["symbol"]),
                        Bitfinex2QA_FREQUENCY_DICT[frequency],
                        QA_util_timestamp_to_str(
                            missing_data_list[i][expected]

Instances