-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbq_load_script
executable file
·27 lines (21 loc) · 1.31 KB
/
bq_load_script
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#!/bin/bash
# make sure bq is installed and run bq once to setup the project
# usage:
#
# ./bg_load_script <dataset> <storage bucket for csvs>
#
# example:
#
# ./bq_load_script jrc_mus_pancreas_1 clio_private_signatures/jrc_mus_pancreas_1
#
# make dataset
bq mk ${1}_imgsearch
# make tables
bq --dataset_id=${1}_imgsearch mk --require_partition_filter --range_partitioning=part,0,4000,1 ${1}_imgsearch.hamming0 "part:integer,signature:integer,x:integer,y:integer,z:integer"
bq --dataset_id=${1}_imgsearch mk --require_partition_filter --range_partitioning=part,0,4000,1 ${1}_imgsearch.hamming1 "part:integer,signature:integer,x:integer,y:integer,z:integer"
bq --dataset_id=${1}_imgsearch mk --require_partition_filter --range_partitioning=part,0,4000,1 ${1}_imgsearch.hamming2 "part:integer,signature:integer,x:integer,y:integer,z:integer"
bq --dataset_id=${1}_imgsearch mk --require_partition_filter --range_partitioning=part,0,4000,1 ${1}_imgsearch.hamming3 "part:integer,signature:integer,x:integer,y:integer,z:integer"
bq load --source_format=CSV ${1}_imgsearch.hamming0 gs://${2}/hamming0/sigs_*
bq load --source_format=CSV ${1}_imgsearch.hamming1 gs://${2}/hamming1/sigs_*
bq load --source_format=CSV ${1}_imgsearch.hamming2 gs://${2}/hamming2/sigs_*
bq load --source_format=CSV ${1}_imgsearch.hamming3 gs://${2}/hamming3/sigs_*