Cargo.toml

[package]
name = "qsv"
version = "2.0.0" #:version
authors = ["Joel Natividad <joel@datHere.com>"]
description = "A Blazing-Fast Data-wrangling toolkit."
documentation = "https://github.com/dathere/qsv#qsv-ultra-fast-csv-data-wrangling-toolkit"
homepage = "https://qsv.dathere.com"
repository = "https://github.com/dathere/qsv"
readme = "README.md"
keywords = ["csv", "geocode", "data-engineering", "etl", "opendata"]
categories = ["command-line-utilities", "parser-implementations"]
license = "MIT OR Unlicense"
autotests = false
edition = "2021"
rust-version = "1.83"
autobins = false
include = [
    "src/**/*",
    "LICENSE-MIT",
    "UNLICENSE",
    "README.md",
    "CHANGELOG.md",
    "resources/luau/vendor/luadate/date.lua",
]

[[bin]]
name              = "qsv"
test              = true
bench             = false
doctest           = false
path              = "src/main.rs"
required-features = ["feature_capable"]

[[bin]]
name              = "qsvlite"
test              = true
bench             = false
doctest           = false
path              = "src/mainlite.rs"
required-features = ["lite"]

[[bin]]
name              = "qsvdp"
test              = true
bench             = false
doctest           = false
path              = "src/maindp.rs"
required-features = ["datapusher_plus"]

[[test]]
name = "tests"
path = "tests/tests.rs"

[profile.release]
codegen-units = 1
debug         = false
lto           = true
opt-level     = 3
strip         = true

[profile.release-samply]
inherits = "release"
debug    = true
strip    = false

[profile.release-nightly]
inherits = "release"
panic    = "abort"

[dependencies]
ahash = "0.8"
arboard = { version = "3.4.1", default-features = false, optional = true }
atoi_simd = "0.16"
base62 = { version = "2.2", optional = true }
bytemuck = { version = "1.21", features = [
    "latest_stable_rust",
], optional = true }
byteorder = "1.5"
bytes = "1"
cached = { version = "0.54", features = [
    "ahash",
    "disk_store",
    "redis_ahash",
], optional = true }
calamine = { version = "0.26", features = ["dates"] }
censor = { version = "0.3", optional = true }
chrono = { version = "0.4", default-features = false }
chrono-tz = "0.10"
console = { version = "0.15", optional = true }
cpc = { version = "1.9", optional = true }
crc32fast = { version = "1.4", optional = true }
crossbeam-channel = "0.5"
csv = "1.3"
csv-core = "0.1"
csv-diff = "0.1.0"
csv-index = "0.1"
csvlens = { version = "0.11", optional = true, default-features = false, features = [
    "clipboard",
] }
csvs_convert = { version = "0.9.0", default-features = false, features = [
    "converters",
], optional = true }
data-encoding = { version = "2.6", optional = true }
dotenvy = "0.15"
dynfmt2 = { version = "0.2", default-features = false, features = ["curly"] }
eudex = { version = "0.1", optional = true }
ext-sort = { version = "0.1", default-features = false }
fast-float2 = "0.2"
flate2 = { version = "1", optional = true }
file-format = { version = "0.26", features = ["reader"] }
filetime = "0.2"
flexi_logger = { version = "0.29", features = [
    "async",
    "compress",
    "dont_minimize_extra_stacks",
], default-features = false }
futures = "0.3"
futures-util = "0.3"
gender_guesser = { version = "0.2", optional = true }
geosuggest-core = { version = "0.6", optional = true }
geosuggest-utils = { version = "0.6", optional = true }
governor = { version = "0.8", optional = true }
grex = { version = "1.4", default-features = false }
gzp = { version = "0.11", default-features = false, features = [
    "snappy_default",
] }
hashbrown = { version = "0.15", optional = true }
human-panic = "2"
indexmap = "2.7"
indicatif = "0.17"
itertools = "0.14"
itoa = "1"
jaq-interpret = "1.5.0"
jaq-parse = "1.0.3"
jemallocator = { version = "0.5", optional = true }
json-objects-to-csv = "0.1.3"
jsonschema = { version = "0.28", features = [
    "resolve-file",
    "resolve-http",
], default-features = false }
local-encoding = { version = "0.2", optional = true }
localzone = { version = "0.3", features = ["auto_validation"] }
log = "0.4"
mimalloc = { version = "0.1", default-features = false, optional = true }
minijinja = { version = "2", features = [
    "json",
    "loop_controls",
    "speedups",
    "stacker",
    "urlencode",
] }
minijinja-contrib = { version = "2", features = [
    "datetime",
    "pycompat",
    "rand",
    "timezone",
    "unicode_wordwrap",
    "wordcount",
    "wordwrap",
] }
mlua = { version = "0.10", features = [
    "luau",
    "luau-jit",
    "serialize",
], optional = true }
num_cpus = "1"
odht = "0.3"
phf = { version = "0.11", features = ["macros"], optional = true }
polars = { version = "0.45", features = [
    "asof_join",
    "avro",
    "avx512",
    "aws",
    "binary_encoding",
    "business",
    # "cloud",
    "coalesce",
    "cross_join",
    "decompress",
    "dtype-categorical",
    # "dtype-full",
    "extract_jsonpath",
    "iejoin",
    "ipc",
    "json",
    "lazy",
    "object",
    "parquet",
    "performant",
    "pivot",
    "semi_anti_join",
    "serde-lazy",
    "sql",
    "streaming",
    "timezones",
], optional = true }
polars-ops = { version = "0.45", optional = true }
publicsuffix = { version = "2.2", optional = true }
pyo3 = { version = "0.23", features = ["auto-initialize"], optional = true }
qsv-dateparser = "0.12"
qsv_docopt = "1.8"
qsv-stats = "0.25"
qsv_currency = "0.7"
qsv-sniffer = { version = "0.10", default-features = false, features = [
    "runtime-dispatch-simd",
] }
qsv_vader_sentiment_analysis = { version = "0.2", optional = true }
rand = "0.8"
rand_hc = "0.3"
rand_xoshiro = "0.6"
rayon = "1.10"
redis = { version = "=0.27", features = [
    "ahash",
    "tcp_nodelay",
], default-features = false, optional = true }
regex = "1"
reqwest = { version = "0.12", features = [
    "blocking",
    "brotli",
    "cookies",
    "deflate",
    "gzip",
    "http2",
    "json",
    "rustls-tls",
    "stream",
    "zstd",
], default-features = false }
rfd = { version = "0.15", optional = true }
rust_decimal = { version = "1.36", default-features = false }
ryu = "1"
sanitize-filename = { version = "0.6", optional = true }
sha256 = "1.5"
simd-json = "0.14"
self_update = { version = "0.42", features = [
    "archive-zip",
    "compression-zip-deflate",
    "rustls",
    "signatures",
], default-features = false, optional = true }
semver = "1"
serde = { version = "1", features = ["derive"] }
serde_json = { version = "1", features = ["preserve_order"] }
serde_urlencoded = { version = "0.7", optional = true }
simple-home-dir = "0.4"
sled = { version = "0.34", optional = true }
smallvec = "1"
snap = "1"
strsim = { version = "0.11", optional = true }
strum = { version = "0.26", features = ["phf"] }
strum_macros = "0.26"
sysinfo = "0.33"
tabwriter = "1.4"
tempfile = "3"
thousands = { version = "0.2", optional = true }
threadpool = "1.8"
titlecase = { version = "3", optional = true }
tokio = { version = "1", features = ["parking_lot", "rt-multi-thread"] }
uuid = { version = "1", features = ["v4", "v7"] }
url = "2.5"
whatlang = { version = "0.16", optional = true }
xxhash-rust = { version = "0.8", features = ["xxh3"] }

# enable parking_lot hardware lock elision on x86_64
[target.'cfg(target_arch = "x86_64")'.dependencies]
parking_lot = { version = "0.12", features = ["hardware-lock-elision"] }
# otherwise, on non-x86_64, use the default
[target.'cfg(not(target_arch = "x86_64"))'.dependencies]
parking_lot = "0.12"

[target.'cfg(not(target_arch = "aarch64"))'.dependencies]
simdutf8 = "0.1"
# use SIMD on aarch64 (Apple Silicon, Raspberry Pi 4, etc.)
[target.'cfg(target_arch = "aarch64")'.dependencies]
simdutf8 = { version = "0.1", features = ["aarch64_neon"] }

[dev-dependencies]
actix-governor = "0.8"
actix-web = { version = "4.9", default-features = false, features = [
    "compress-brotli",
    "compress-gzip",
] }
assert-json-diff = "2.0"
newline-converter = "0.3"

# disable these dev dependencies for testing the `to` command
# as they are expensive and slow down the build/CI tests
# postgres = "0.19"
# rusqlite = { version = "0.32", features = ["bundled"] }

quickcheck  = { version = "1", default-features = false }
serial_test = { version = "3.2", features = ["file_locks"] }

[patch.crates-io]
# use our tweaked fork of csv crate
# the csv crate underpins a lot of qsv's functionality, so every perf tweak helps
# the main feature of this csv fork is accelerated UTF-8 validation using the simdutf8 crate
# non-allocating ByteRecord trim and a non-allocating StringRecord trim_ascii
csv       = { git = "https://github.com/dathere/rust-csv", branch = "qsv-optimized-non-allocating-byterecord-trim" }
csv-core  = { git = "https://github.com/dathere/rust-csv", branch = "qsv-optimized-non-allocating-byterecord-trim" }
csv-index = { git = "https://github.com/dathere/rust-csv", branch = "qsv-optimized-non-allocating-byterecord-trim" }

# use cached with bumped redis using 0.27
cached = { git = "https://github.com/jaemk/cached", rev = "bbf457e" }

# use our calamine fork with unreleased fixes, an upgraded quick-xml, and accelerated int/float parsing
calamine = { git = "https://github.com/jqnatividad/calamine", branch = "qsv-2-optimized" }

# use our fork of csvlens with unreleased fixes, bumped dependencies, and new clap-less API
csvlens = { git = "https://github.com/jqnatividad/csvlens", branch = "116-clap-optional-dependency-cli-feature" }

# needed to get latest dependencies and unreleased fixes
grex = { git = "https://github.com/pemistahl/grex", rev = "cb71c10" }

# use modernized version of local_encoding
local-encoding = { git = "https://github.com/slonopotamus/local-encoding-rs", branch = "travis-madness" }

# use our patched fork of sled to get rid of unmaintained instant
sled = { git = "https://github.com/dathere/sled", branch = "v0.34.7-bumped-parking_lot_to_0.12" }

# use our fork of latest strum upstream with bumped phf dependency and unreleased fixes.
strum        = { git = "https://github.com/dathere/strum", branch = "upstream-with-bumped-phf-to-0.11" }
strum_macros = { git = "https://github.com/dathere/strum", branch = "upstream-with-bumped-phf-to-0.11" }

# use textwrap upstream with unreleased fixes and dependencies, used by minijinja-contrib
textwrap = { git = "https://github.com/mgeisler/textwrap", rev = "0e581b8" }

# Polars has a much higher release tempo for its Python bindings compared
# to its underlying Rust library. See https://github.com/pola-rs/polars/releases
# It's qsv's policy to use the latest upstream of polars/py-polars
# to take advantage of Polars' latest unreleased fixes and features.
# Based on what's available at the time of qsv's release, we may need to pin polars to a py-polars tag
# or a specific commit if more revisions have been made since the latest polars/py-polars release.
# BUILD NOTE: Be sure to set QSV_POLARS_REV below to the latest commit short hash or tag 
# of polars/py-polars before building qsv. This allows us to show the polars rev/tag in --version.
# The convention to use for QSV_POLARS_REV is:
# - if we are using a release version of Rust Polars, leave QSV_POLARS_REV empty
# - if we are using a release version of Python Polars, use the Python tag
#   (e.g. py-1.19.0)
# - if we are using a commit hash, separate the tag and commit hash with a colon, in the order of
#   - Python Polars tag
#   - short commit hash
#     (e.g. py-1.19.0:52ea381 to indicate that we are Python Polars 1.19.0,
#      and the commit hash 52ea381)
# ================================
# QSV_POLARS_REV=py-1.19.0:2e0b3a3
# polars = { git = "https://github.com/pola-rs/polars", tag = "py-1.19.0" }
polars = { git = "https://github.com/pola-rs/polars", rev = "2e0b3a3" }
# polars-ops = { git = "https://github.com/pola-rs/polars", tag = "py-1.19.0" }
polars-ops = { git = "https://github.com/pola-rs/polars", rev = "2e0b3a3" }

[features]
default = ["mimalloc"]
distrib_features = [
    "feature_capable",
    "apply",
    "fetch",
    "foreach",
    "geocode",
    "luau",
    "polars",
    "python",
    "to",
]
all_features = ["distrib_features", "self_update", "ui"]
apply = [
    "base62",
    "censor",
    "cpc",
    "crc32fast",
    "data-encoding",
    "eudex",
    "gender_guesser",
    "hashbrown",
    "qsv_vader_sentiment_analysis",
    "strsim",
    "thousands",
    "titlecase",
    "whatlang",
]
clipboard = ["arboard"]
fetch = [
    "cached",
    "console",
    "flate2",
    "governor",
    "hashbrown",
    "publicsuffix",
    "redis",
    "serde_urlencoded",
    "sled",
]
foreach = ["local-encoding"]
geocode = [
    "bytemuck",
    "cached",
    "geosuggest-core",
    "geosuggest-utils",
    "phf",
    "sled",
]
luau = ["mlua", "sanitize-filename"]
polars = ["dep:polars", "bytemuck", "dep:polars-ops"]
prompt = ["rfd"]
python = ["pyo3"]
to = ["csvs_convert"]
lens = ["csvlens"]
lite = []
datapusher_plus = ["self_update"]
ui = ["clipboard", "prompt", "lens"]
feature_capable = []
nightly = [
    "crc32fast/nightly",
    "hashbrown/nightly",
    "polars/nightly",
    "polars/simd",
    "pyo3/nightly",
    "rand/nightly",
]

[package.metadata.deb]
maintainer           = "Konstantin Sivakov <konstantin@datHere.com>"
copyright            = "2024, datHere Inc. <www.dathere.com>"
extended-description = """A high performance CSV data-wrangling toolkit."""
depends              = "$auto"
section              = "utility"
priority             = "optional"

# Default feature and asset
features = ["feature_capable"]
assets   = [["target/release/qsv", "/usr/local/bin/", "755"]]

# Conditional features and assets
[package.metadata.deb.variants.lite]
features = ["lite"]
assets   = [["target/release/qsvlite", "/usr/local/bin/", "755"]]

[package.metadata.deb.variants.datapusher_plus]
features = ["datapusher_plus", "luau"]
assets   = [["target/release/qsvdp", "/usr/local/bin/", "755"]]