Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix bugs in win11 #383

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

fix bugs in win11 #383

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
eda2bf5
fix bugs in win11
tianjixuetu Dec 19, 2024
02349d4
fix format
cloudQuant Dec 19, 2024
f6a4d47
update donwload daily kline to multi-threaded
cloudQuant Dec 19, 2024
851caf3
fix multi-threaded bugs
cloudQuant Dec 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 76 additions & 71 deletions python/download-aggTrade.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,102 +14,107 @@
import pandas as pd
from enums import *
from utility import download_file, get_all_symbols, get_parser, get_start_end_date_objects, convert_to_date_object, \
get_path
get_path


def download_monthly_aggTrades(trading_type, symbols, num_symbols, years, months, start_date, end_date, folder, checksum):
current = 0
date_range = None
def download_monthly_aggTrades(trading_type, symbols, num_symbols, years, months, start_date, end_date, folder,
checksum):
current = 0
date_range = None

if start_date and end_date:
date_range = start_date + " " + end_date
if start_date and end_date:
date_range = start_date + " " + end_date

if not start_date:
start_date = START_DATE
else:
start_date = convert_to_date_object(start_date)
if not start_date:
start_date = START_DATE
else:
start_date = convert_to_date_object(start_date)

if not end_date:
end_date = END_DATE
else:
end_date = convert_to_date_object(end_date)

print("Found {} symbols".format(num_symbols))

if not end_date:
end_date = END_DATE
else:
end_date = convert_to_date_object(end_date)
for symbol in symbols:
print("[{}/{}] - start download monthly {} aggTrades ".format(current + 1, num_symbols, symbol))
for year in years:
for month in months:
current_date = convert_to_date_object('{}-{}-01'.format(year, month))
if current_date >= start_date and current_date <= end_date:
path = get_path(trading_type, "aggTrades", "monthly", symbol)
file_name = "{}-aggTrades-{}-{}.zip".format(symbol.upper(), year, '{:02d}'.format(month))
download_file(path, file_name, date_range, folder)

print("Found {} symbols".format(num_symbols))
if checksum == 1:
checksum_path = get_path(trading_type, "aggTrades", "monthly", symbol)
checksum_file_name = "{}-aggTrades-{}-{}.zip.CHECKSUM".format(symbol.upper(), year,
'{:02d}'.format(month))
download_file(checksum_path, checksum_file_name, date_range, folder)

for symbol in symbols:
print("[{}/{}] - start download monthly {} aggTrades ".format(current+1, num_symbols, symbol))
for year in years:
for month in months:
current_date = convert_to_date_object('{}-{}-01'.format(year, month))
if current_date >= start_date and current_date <= end_date:
path = get_path(trading_type, "aggTrades", "monthly", symbol)
file_name = "{}-aggTrades-{}-{}.zip".format(symbol.upper(), year, '{:02d}'.format(month))
download_file(path, file_name, date_range, folder)
current += 1

if checksum == 1:
checksum_path = get_path(trading_type, "aggTrades", "monthly", symbol)
checksum_file_name = "{}-aggTrades-{}-{}.zip.CHECKSUM".format(symbol.upper(), year, '{:02d}'.format(month))
download_file(checksum_path, checksum_file_name, date_range, folder)

current += 1

def download_daily_aggTrades(trading_type, symbols, num_symbols, dates, start_date, end_date, folder, checksum):
current = 0
date_range = None
current = 0
date_range = None

if start_date and end_date:
date_range = start_date + " " + end_date
if start_date and end_date:
date_range = start_date + " " + end_date

if not start_date:
start_date = START_DATE
else:
start_date = convert_to_date_object(start_date)
if not start_date:
start_date = START_DATE
else:
start_date = convert_to_date_object(start_date)

if not end_date:
end_date = END_DATE
else:
end_date = convert_to_date_object(end_date)

if not end_date:
end_date = END_DATE
else:
end_date = convert_to_date_object(end_date)
print("Found {} symbols".format(num_symbols))

print("Found {} symbols".format(num_symbols))
for symbol in symbols:
print("[{}/{}] - start download daily {} aggTrades ".format(current + 1, num_symbols, symbol))
for date in dates:
current_date = convert_to_date_object(date)
if current_date >= start_date and current_date <= end_date:
path = get_path(trading_type, "aggTrades", "daily", symbol)
file_name = "{}-aggTrades-{}.zip".format(symbol.upper(), date)
download_file(path, file_name, date_range, folder)

for symbol in symbols:
print("[{}/{}] - start download daily {} aggTrades ".format(current+1, num_symbols, symbol))
for date in dates:
current_date = convert_to_date_object(date)
if current_date >= start_date and current_date <= end_date:
path = get_path(trading_type, "aggTrades", "daily", symbol)
file_name = "{}-aggTrades-{}.zip".format(symbol.upper(), date)
download_file(path, file_name, date_range, folder)
if checksum == 1:
checksum_path = get_path(trading_type, "aggTrades", "daily", symbol)
checksum_file_name = "{}-aggTrades-{}.zip.CHECKSUM".format(symbol.upper(), date)
download_file(checksum_path, checksum_file_name, date_range, folder)

if checksum == 1:
checksum_path = get_path(trading_type, "aggTrades", "daily", symbol)
checksum_file_name = "{}-aggTrades-{}.zip.CHECKSUM".format(symbol.upper(), date)
download_file(checksum_path, checksum_file_name, date_range, folder)
current += 1

current += 1

if __name__ == "__main__":
parser = get_parser('aggTrades')
args = parser.parse_args(sys.argv[1:])

if not args.symbols:
print("fetching all symbols from exchange")
symbols = get_all_symbols(args.type)
num_symbols = len(symbols)
print("fetching all symbols from exchange")
symbols = get_all_symbols(args.type)
num_symbols = len(symbols)
else:
symbols = args.symbols
num_symbols = len(symbols)
print("fetching {} symbols from exchange".format(num_symbols))
symbols = args.symbols
num_symbols = len(symbols)
print("fetching {} symbols from exchange".format(num_symbols))

if args.dates:
dates = args.dates
dates = args.dates
else:
period = convert_to_date_object(datetime.today().strftime('%Y-%m-%d')) - convert_to_date_object(
PERIOD_START_DATE)
dates = pd.date_range(end=datetime.today(), periods=period.days + 1).to_pydatetime().tolist()
dates = [date.strftime("%Y-%m-%d") for date in dates]
if args.skip_monthly == 0:
download_monthly_aggTrades(args.type, symbols, num_symbols, args.years, args.months, args.startDate, args.endDate, args.folder, args.checksum)
period = convert_to_date_object(datetime.today().strftime('%Y-%m-%d')) - convert_to_date_object(
PERIOD_START_DATE)
dates = pd.date_range(end=datetime.today(), periods=period.days + 1).to_pydatetime().tolist()
dates = [date.strftime("%Y-%m-%d") for date in dates]
if args.skip_monthly == 0:
download_monthly_aggTrades(args.type, symbols, num_symbols, args.years, args.months, args.startDate,
args.endDate, args.folder, args.checksum)
if args.skip_daily == 0:
download_daily_aggTrades(args.type, symbols, num_symbols, dates, args.startDate, args.endDate, args.folder, args.checksum)

download_daily_aggTrades(args.type, symbols, num_symbols, dates, args.startDate, args.endDate, args.folder,
args.checksum)
11 changes: 6 additions & 5 deletions python/download-futures-markPriceKlines.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@


def download_monthly_markPriceKlines(trading_type, symbols, num_symbols, intervals, years, months, start_date,
end_date, folder, checksum):
end_date, folder, checksum):
current = 0
date_range = None

Expand Down Expand Up @@ -58,7 +58,7 @@ def download_monthly_markPriceKlines(trading_type, symbols, num_symbols, interva


def download_daily_markPriceKlines(trading_type, symbols, num_symbols, intervals, dates, start_date, end_date, folder,
checksum):
checksum):
current = 0
date_range = None

Expand Down Expand Up @@ -115,10 +115,11 @@ def download_daily_markPriceKlines(trading_type, symbols, num_symbols, intervals
if args.dates:
dates = args.dates
else:
period = convert_to_date_object(datetime.today().strftime('%Y-%m-%d')) - convert_to_date_object(PERIOD_START_DATE)
period = convert_to_date_object(datetime.today().strftime('%Y-%m-%d')) - convert_to_date_object(
PERIOD_START_DATE)
dates = pd.date_range(end=datetime.today(), periods=period.days + 1).to_pydatetime().tolist()
dates = [date.strftime("%Y-%m-%d") for date in dates]
download_monthly_markPriceKlines(args.type, symbols, num_symbols, args.intervals, args.years, args.months,
args.startDate, args.endDate, args.folder, args.checksum)
args.startDate, args.endDate, args.folder, args.checksum)
download_daily_markPriceKlines(args.type, symbols, num_symbols, args.intervals, dates, args.startDate,
args.endDate, args.folder, args.checksum)
args.endDate, args.folder, args.checksum)
11 changes: 6 additions & 5 deletions python/download-futures-premiumIndexKlines.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@


def download_monthly_premiumIndexKlines(trading_type, symbols, num_symbols, intervals, years, months, start_date,
end_date, folder, checksum):
end_date, folder, checksum):
current = 0
date_range = None

Expand Down Expand Up @@ -57,8 +57,9 @@ def download_monthly_premiumIndexKlines(trading_type, symbols, num_symbols, inte
current += 1


def download_daily_premiumIndexKlines(trading_type, symbols, num_symbols, intervals, dates, start_date, end_date, folder,
checksum):
def download_daily_premiumIndexKlines(trading_type, symbols, num_symbols, intervals, dates, start_date, end_date,
folder,
checksum):
current = 0
date_range = None

Expand Down Expand Up @@ -120,6 +121,6 @@ def download_daily_premiumIndexKlines(trading_type, symbols, num_symbols, interv
dates = pd.date_range(end=datetime.today(), periods=period.days + 1).to_pydatetime().tolist()
dates = [date.strftime("%Y-%m-%d") for date in dates]
download_monthly_premiumIndexKlines(args.type, symbols, num_symbols, args.intervals, args.years, args.months,
args.startDate, args.endDate, args.folder, args.checksum)
args.startDate, args.endDate, args.folder, args.checksum)
download_daily_premiumIndexKlines(args.type, symbols, num_symbols, args.intervals, dates, args.startDate,
args.endDate, args.folder, args.checksum)
args.endDate, args.folder, args.checksum)
Loading