From 2f51455ee065a0e738d7b4b21996d13cc6c44d94 Mon Sep 17 00:00:00 2001 From: Mudit Gupta Date: Sat, 1 Aug 2020 11:54:20 +0530 Subject: [PATCH 1/6] initial support --- README.md | 29 +++++---- h8mail/requirements.txt | 1 + h8mail/utils/classes.py | 2 +- h8mail/utils/localzstdsearch.py | 111 ++++++++++++++++++++++++++++++++ h8mail/utils/run.py | 20 ++++++ setup.py | 6 +- 6 files changed, 153 insertions(+), 16 deletions(-) create mode 100644 h8mail/utils/localzstdsearch.py diff --git a/README.md b/README.md index af87183..513bff4 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ [![platforms](https://img.shields.io/badge/platforms-Windows%20%7C%20Linux%20%7C%20OSX-success.svg)](https://pypi.org/project/h8mail/) [![PyPI version](https://badge.fury.io/py/h8mail.svg)](https://badge.fury.io/py/h8mail) -[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/h8mail.svg)](https://pypi.org/project/h8mail/) [![Downloads](https://pepy.tech/badge/h8mail)](https://pepy.tech/project/h8mail) [![travis](https://img.shields.io/travis/khast3x/h8mail.svg)](https://travis-ci.org/khast3x/h8mail) -[![Docker Pulls](https://img.shields.io/docker/pulls/kh4st3x00/h8mail.svg)](https://hub.docker.com/r/kh4st3x00/h8mail) [![MicroBadger Size (tag)](https://img.shields.io/microbadger/image-size/kh4st3x00/h8mail.svg?color=ok)](https://hub.docker.com/r/kh4st3x00/h8mail/builds) -**h8mail** is an email OSINT and breach hunting tool using [different breach and reconnaissance services](#apis), or local breaches such as Troy Hunt's "Collection1" and the infamous "Breach Compilation" torrent. +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/h8mail.svg)](https://pypi.org/project/h8mail/) [![Downloads](https://pepy.tech/badge/h8mail)](https://pepy.tech/project/h8mail) [![travis](https://img.shields.io/travis/khast3x/h8mail.svg)](https://travis-ci.org/khast3x/h8mail) +[![Docker Pulls](https://img.shields.io/docker/pulls/kh4st3x00/h8mail.svg)](https://hub.docker.com/r/kh4st3x00/h8mail) [![MicroBadger Size (tag)](https://img.shields.io/microbadger/image-size/kh4st3x00/h8mail.svg?color=ok)](https://hub.docker.com/r/kh4st3x00/h8mail/builds) +**h8mail** is an email OSINT and breach hunting tool using [different breach and reconnaissance services](#apis), or local breaches such as Troy Hunt's "Collection1" and the infamous "Breach Compilation" torrent. ---- @@ -38,11 +38,11 @@ * :mag_right: Email pattern matching (reg exp), useful for reading from other tool outputs * :earth_africa: Pass URLs to directly find and target emails in pages * :dizzy: Loosey patterns for local searchs ("john.smith", "evilcorp") -* :package: Painless install. Available through `pip`, only requires `requests` +* :package: Painless install. Available through `pip`, only requires `requests` and `zstandard` * :white_check_mark: Bulk file-reading for targeting * :memo: Output to CSV file * :muscle: Compatible with the "Breach Compilation" torrent scripts -* :house: Search cleartext and compressed .gz files locally using multiprocessing +* :house: Search cleartext and compressed .gz, .zst files locally using multiprocessing * :cyclone: Compatible with "Collection#1" * :fire: Get related emails * :dragon_face: Chase related emails by adding them to the ongoing search @@ -75,7 +75,7 @@ | [Dehashed.sh](https://dehashed.com/) - Service | Cleartext passwords, hashs and salts, usernames, IPs, domain | :construction: :key: | | :new: [IntelX.io](https://intelx.io/signup) - Service (free trial) | Cleartext passwords, hashs and salts, usernames, IPs, domain, Bitcoin Wallets, IBAN | :white_check_mark: :key: | -*:key: - API key required* +*:key: - API key required* @@ -90,10 +90,11 @@ usage: h8mail [-h] [-t USER_TARGETS [USER_TARGETS ...]] [-c CONFIG_FILE [CONFIG_FILE ...]] [-o OUTPUT_FILE] [-bc BC_PATH] [-sk] [-k CLI_APIKEYS [CLI_APIKEYS ...]] [-lb LOCAL_BREACH_SRC [LOCAL_BREACH_SRC ...]] - [-gz LOCAL_GZIP_SRC [LOCAL_GZIP_SRC ...]] [-sf] + [-gz LOCAL_GZIP_SRC [LOCAL_GZIP_SRC ...]] + [-zs LOCAL_ZSTD_SRC [LOCAL_ZSTD_SRC ...]] [-sf] [-ch [CHASE_LIMIT]] [--power-chase] [--hide] [--debug] [--gen-config] - + Email information and password lookup tool optional arguments: @@ -136,6 +137,11 @@ optional arguments: targets. Uses multiprocesses, one separate process per file. Supports file or folder as input, and filepath globing. Looks for 'gz' in filename + -zs LOCAL_ZSTD_SRC [LOCAL_ZSTD_SRC ...], --zstd LOCAL_ZSTD_SRC [LOCAL_ZSTD_SRC ...] + Local zst (zstandard) compressed breaches to scans for + targets. Uses multiprocesses, one separate process per + file. Supports file or folder as input, and filepath + globing. Looks for 'zst' in filename -sf, --single-file If breach contains big cleartext or tar.gz files, set this flag to view the progress bar. Disables concurrent file searching for stability @@ -218,10 +224,10 @@ $ h8mail -u "https://pastebin.com/raw/kQ6WNKqY" "list_of_urls.txt" * [Snusbase](https://snusbase.com/) for being developer friendly * [kodykinzie](https://twitter.com/kodykinzie) for making a nice [introduction and walkthrough article](https://null-byte.wonderhowto.com/how-to/exploit-recycled-credentials-with-h8mail-break-into-user-accounts-0188600/) and [video](https://www.youtube.com/watch?v=z8G_vBBHtfA) on installing and using h8mail * [Leak-Lookup](https://leak-lookup.com/) for being developer friendly -* [WeLeakInfo](https://weleakinfo.com/) for being developer friendly +* [WeLeakInfo](https://weleakinfo.com/) for being developer friendly * h8mail's Pypi integration is strongly based on the work of audreyr's [CookieCutter PyPackage](https://github.com/audreyr/cookiecutter-pypackage) * Logo generated using Hatchful by Shopify -* [Jake Creps](https://twitter.com/jakecreps) for his [h8mail v2 introduction](https://jakecreps.com/2019/06/21/h8mail/) +* [Jake Creps](https://twitter.com/jakecreps) for his [h8mail v2 introduction](https://jakecreps.com/2019/06/21/h8mail/) * [Alejandro Caceres](https://twitter.com/_hyp3ri0n) for making scylla.sh available. Be sure to [support](https://www.buymeacoffee.com/Eiw47ImnT) him if you can * [IntelX](https://intelx.io) for being developer friendly @@ -250,7 +256,7 @@ $ h8mail -u "https://pastebin.com/raw/kQ6WNKqY" "list_of_urls.txt" * Service providers that wish being integrated can send me an email at `k at khast3x dot club` (PGP friendly) * h8mail is maintained on my free time. Feedback and war stories are welcomed. * Licence is BSD 3 clause -* My code is [signed](https://help.github.com/en/articles/signing-commits) with my [Keybase](https://keybase.io/ktx) PGP key. You can get it using: +* My code is [signed](https://help.github.com/en/articles/signing-commits) with my [Keybase](https://keybase.io/ktx) PGP key. You can get it using: ```bash # curl + gpg pro tip: import ktx's keys curl https://keybase.io/ktx/pgp_keys.asc | gpg --import @@ -266,4 +272,3 @@ ___

- diff --git a/h8mail/requirements.txt b/h8mail/requirements.txt index f229360..c047178 100644 --- a/h8mail/requirements.txt +++ b/h8mail/requirements.txt @@ -1 +1,2 @@ requests +zstandard diff --git a/h8mail/utils/classes.py b/h8mail/utils/classes.py index 85a51f7..b6a2ed4 100644 --- a/h8mail/utils/classes.py +++ b/h8mail/utils/classes.py @@ -15,7 +15,7 @@ class local_breach_target: Class is called when performing local file search. This class is meant to store found data, to be later appended to the existing target objects. local_to_targets() tranforms to target object - Used by both cleartext and gzip search + Used by cleartext, gzip and zstd search """ def __init__(self, target_data, fp, ln, content): diff --git a/h8mail/utils/localzstdsearch.py b/h8mail/utils/localzstdsearch.py new file mode 100644 index 0000000..c85e3bb --- /dev/null +++ b/h8mail/utils/localzstdsearch.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- + +from multiprocessing import Pool +from .classes import local_breach_target +from .colors import colors as c +from .localsearch import raw_in_count, progress +import io +import os +import sys +import signal +import zstandard as zstd + + +def progress_zstd(count): + """ + Prints count without rewriting to stdout + """ + sys.stdout.write("Lines checked:%i\r" % (count)) + sys.stdout.write("\033[K") + + +def zstd_worker(filepath, target_list): + """ + Searches for every email from target_list in every line of filepath. + Uses python zstandard bindings to decompress file line by line. + Archives with multiple files are read as long single files. + Attempts to decode line using cp437. If it fails, catch and use raw data. + """ + try: + found_list = [] + size = os.stat(filepath).st_size + with open(filepath, "rb") as breach_file: + dctx = zstd.ZstdDecompressor() + stream_reader = dctx.stream_reader(breach_file) + zstdfile = io.TextIOWrapper(stream_reader, encoding='utf-8', errors='replace') + c.info_news( + "Worker [{PID}] is searching for targets in {filepath} ({size:,.0f} MB)".format( + PID=os.getpid(), filepath=filepath, size=size / float(1 << 20) + ) + ) + for cnt, line in enumerate(zstdfile): + for t in target_list: + if t in str(line): + c.good_news( + f"Found occurrence [{filepath}] Line {cnt}: {line}" + ) + found_list.append( + local_breach_target(t, filepath, cnt, str(line)) + ) + return found_list + except Exception as e: + c.bad_news("Something went wrong with zstd worker") + print(e) + + +def local_zstd_search(files_to_parse, target_list): + """ + Receives list of all files to check for target_list. + Starts a worker pool, one worker per file. + Return list of local_breach_targets objects to be tranformed in target objects. + """ + original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) + pool = Pool() + found_list = [] + signal.signal(signal.SIGINT, original_sigint_handler) + # Launch + try: + async_results = [ + pool.apply_async(zstd_worker, args=(f, target_list)) + for i, f in enumerate(files_to_parse) + ] + for r in async_results: + if r.get() is not None: + found_list.extend(r.get(60)) + except KeyboardInterrupt: + c.bad_news("Caught KeyboardInterrupt, terminating workers") + pool.terminate() + else: + c.info_news("Terminating worker pool") + pool.close() + pool.join() + return found_list + + +def local_search_single_zstd(files_to_parse, target_list): + """ + Single process searching of every target_list emails, in every files_to_parse list. + To be used when stability for big files is a priority + Return list of local_breach_target objects to be tranformed in target objects + """ + found_list = [] + for file_to_parse in files_to_parse: + with open(file_to_parse, "rb") as breach_file: + dctx = zstd.ZstdDecompressor() + stream_reader = dctx.stream_reader(breach_file) + zstdfile = io.TextIOWrapper(stream_reader, encoding= 'utf-8', errors='replace') + size = os.stat(file_to_parse).st_size + c.info_news( + f"Searching for targets in {file_to_parse} ({size} bytes)" + ) + for cnt, line in enumerate(zstdfile): + progress_zstd(cnt) + for t in target_list: + if t in str(line): + c.good_news( + f"Found occurrence [{file_to_parse}] Line {cnt}: {line}" + ) + found_list.append( + local_breach_target(t, file_to_parse, cnt, str(line)) + ) + return found_list diff --git a/h8mail/utils/run.py b/h8mail/utils/run.py index 32905f3..9dbe8fd 100644 --- a/h8mail/utils/run.py +++ b/h8mail/utils/run.py @@ -22,6 +22,7 @@ ) from .localsearch import local_search, local_search_single, local_to_targets from .localgzipsearch import local_gzip_search, local_search_single_gzip +from .localzstdsearch import local_zstd_search, local_search_single_zstd from .summary import print_summary from .chase import chase from .print_results import print_results @@ -210,6 +211,18 @@ def h8mail(user_args): breached_targets = local_to_targets( breached_targets, local_found, user_args ) + # Handle zstd search + if user_args.local_zstd_src: + for arg in user_args.local_zstd_src: + res = find_files(arg, "zst") + if user_args.single_file: + local_found = local_search_single_zstd(res, targets) + else: + local_found = local_zstd_search(res, targets) + if local_found is not None: + breached_targets = local_to_targets( + breached_targets, local_found, user_args + ) print_results(breached_targets, user_args.hide) @@ -299,6 +312,13 @@ def parse_args(args): help="Local tar.gz (gzip) compressed breaches to scans for targets. Uses multiprocesses, one separate process per file. Supports file or folder as input, and filepath globing. Looks for 'gz' in filename", nargs="+", ) + parser.add_argument( + "-zs", + "--zstd", + dest="local_zstd_src", + help="Local zst (zstandard) compressed breaches to scans for targets. Uses multiprocesses, one separate process per file. Supports file or folder as input, and filepath globing. Looks for 'zst' in filename", + nargs="+", + ) parser.add_argument( "-sf", "--single-file", diff --git a/setup.py b/setup.py index 574f225..a9077b6 100644 --- a/setup.py +++ b/setup.py @@ -15,11 +15,11 @@ # with open("HISTORY.rst") as history_file: # history = history_file.read() -requirements = ["requests"] +requirements = ["requests", "zstandard"] -setup_requirements = ["requests"] +setup_requirements = ["requests", "zstandard"] -test_requirements = ["requests"] +test_requirements = ["requests", "zstandard"] setup( author="khast3x", From c411379f1aa27cc60e0e0fb8b0aacb4d0dbf8851 Mon Sep 17 00:00:00 2001 From: ktx <1370650+khast3x@users.noreply.github.com> Date: Thu, 28 Jan 2021 10:38:55 +0100 Subject: [PATCH 2/6] Delete README.md --- README.md | 274 ------------------------------------------------------ 1 file changed, 274 deletions(-) delete mode 100644 README.md diff --git a/README.md b/README.md deleted file mode 100644 index 513bff4..0000000 --- a/README.md +++ /dev/null @@ -1,274 +0,0 @@ -

- -

- -[![platforms](https://img.shields.io/badge/platforms-Windows%20%7C%20Linux%20%7C%20OSX-success.svg)](https://pypi.org/project/h8mail/) [![PyPI version](https://badge.fury.io/py/h8mail.svg)](https://badge.fury.io/py/h8mail) -[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/h8mail.svg)](https://pypi.org/project/h8mail/) [![Downloads](https://pepy.tech/badge/h8mail)](https://pepy.tech/project/h8mail) [![travis](https://img.shields.io/travis/khast3x/h8mail.svg)](https://travis-ci.org/khast3x/h8mail) -[![Docker Pulls](https://img.shields.io/docker/pulls/kh4st3x00/h8mail.svg)](https://hub.docker.com/r/kh4st3x00/h8mail) [![MicroBadger Size (tag)](https://img.shields.io/microbadger/image-size/kh4st3x00/h8mail.svg?color=ok)](https://hub.docker.com/r/kh4st3x00/h8mail/builds) -**h8mail** is an email OSINT and breach hunting tool using [different breach and reconnaissance services](#apis), or local breaches such as Troy Hunt's "Collection1" and the infamous "Breach Compilation" torrent. - ----- - - -

- -

- - - ----- - - -## :book: Table of Content - -- [Table of Content](#book-Table-of-Content) -- [Features](#tangerine-Features) - - [APIs](#APIs) -- [Usage](#tangerine-Usage) -- [Usage examples](#tangerine-Usage-examples) -- [Thanks & Credits](#tangerine-Thanks--Credits) -- [Related open source projects](#tangerine-Related-open-source-projects) - - ----- - - -## :tangerine: Features - -* :mag_right: Email pattern matching (reg exp), useful for reading from other tool outputs -* :earth_africa: Pass URLs to directly find and target emails in pages -* :dizzy: Loosey patterns for local searchs ("john.smith", "evilcorp") -* :package: Painless install. Available through `pip`, only requires `requests` and `zstandard` -* :white_check_mark: Bulk file-reading for targeting -* :memo: Output to CSV file -* :muscle: Compatible with the "Breach Compilation" torrent scripts -* :house: Search cleartext and compressed .gz, .zst files locally using multiprocessing - * :cyclone: Compatible with "Collection#1" -* :fire: Get related emails -* :dragon_face: Chase related emails by adding them to the ongoing search -* :crown: Supports premium lookup services for advanced users -* :factory: Custom query premium APIs. Supports username, hash, ip, domain and password and more -* :books: Regroup breach results for all targets and methods -* :eyes: Includes option to hide passwords for demonstrations -* :rainbow: Delicious colors - ---- - -### :package: `pip3 install h8mail` - ------ - - -#### APIs - -| Service | Functions | Status | -|-|-|-| -| [HaveIBeenPwned(v3)](https://haveibeenpwned.com/) | Number of email breaches | :white_check_mark: :key: | -| [HaveIBeenPwned Pastes(v3)](https://haveibeenpwned.com/Pastes) | URLs of text files mentioning targets | :white_check_mark: :key: | -| [Hunter.io](https://hunter.io/) - Public | Number of related emails | :white_check_mark: | -| [Hunter.io](https://hunter.io/) - Service (free tier) | Cleartext related emails, Chasing | :white_check_mark: :key: | -| [Snusbase](https://snusbase.com/) - Service | Cleartext passwords, hashs and salts, usernames, IPs - Fast :zap: | :white_check_mark: :key: | -| [Leak-Lookup](https://leak-lookup.com/) - Public | Number of search-able breach results | :white_check_mark: (:key:) | -| [Leak-Lookup](https://leak-lookup.com/) - Service | Cleartext passwords, hashs and salts, usernames, IPs, domain | :white_check_mark: :key: | -| [Emailrep.io](https://emailrep.io/) - Service (free) | Last seen in breaches, social media profiles | :white_check_mark: :key: | -| [Scylla.sh](https://scylla.sh/) - Service (free) | Cleartext passwords, hashs and salts, usernames, IPs, domain | :white_check_mark: | -| [Dehashed.sh](https://dehashed.com/) - Service | Cleartext passwords, hashs and salts, usernames, IPs, domain | :construction: :key: | -| :new: [IntelX.io](https://intelx.io/signup) - Service (free trial) | Cleartext passwords, hashs and salts, usernames, IPs, domain, Bitcoin Wallets, IBAN | :white_check_mark: :key: | - -*:key: - API key required* - - - - ------ - -## :tangerine: Usage - -```bash -usage: h8mail [-h] [-t USER_TARGETS [USER_TARGETS ...]] - [-u USER_URLS [USER_URLS ...]] [-q USER_QUERY] [--loose] - [-c CONFIG_FILE [CONFIG_FILE ...]] [-o OUTPUT_FILE] - [-bc BC_PATH] [-sk] [-k CLI_APIKEYS [CLI_APIKEYS ...]] - [-lb LOCAL_BREACH_SRC [LOCAL_BREACH_SRC ...]] - [-gz LOCAL_GZIP_SRC [LOCAL_GZIP_SRC ...]] - [-zs LOCAL_ZSTD_SRC [LOCAL_ZSTD_SRC ...]] [-sf] - [-ch [CHASE_LIMIT]] [--power-chase] [--hide] [--debug] - [--gen-config] - -Email information and password lookup tool - -optional arguments: - -h, --help show this help message and exit - -t USER_TARGETS [USER_TARGETS ...], --targets USER_TARGETS [USER_TARGETS ...] - Either string inputs or files. Supports email pattern - matching from input or file, filepath globing and - multiple arguments - -u USER_URLS [USER_URLS ...], --url USER_URLS [USER_URLS ...] - Either string inputs or files. Supports URL pattern - matching from input or file, filepath globing and - multiple arguments. Parse URLs page for emails. - Requires http:// or https:// in URL. - -q USER_QUERY, --custom-query USER_QUERY - Perform a custom query. Supports username, password, - ip, hash, domain. Performs an implicit "loose" search - when searching locally - --loose Allow loose search by disabling email pattern - recognition. Use spaces as pattern seperators - -c CONFIG_FILE [CONFIG_FILE ...], --config CONFIG_FILE [CONFIG_FILE ...] - Configuration file for API keys. Accepts keys from - Snusbase, WeLeakInfo, Leak-Lookup, HaveIBeenPwned, - Emailrep, Dehashed and hunterio - -o OUTPUT_FILE, --output OUTPUT_FILE - File to write CSV output - -bc BC_PATH, --breachcomp BC_PATH - Path to the breachcompilation torrent folder. Uses the - query.sh script included in the torrent - -sk, --skip-defaults Skips HaveIBeenPwned and HunterIO check. Ideal for - local scans - -k CLI_APIKEYS [CLI_APIKEYS ...], --apikey CLI_APIKEYS [CLI_APIKEYS ...] - Pass config options. Supported format: "K=V,K=V" - -lb LOCAL_BREACH_SRC [LOCAL_BREACH_SRC ...], --local-breach LOCAL_BREACH_SRC [LOCAL_BREACH_SRC ...] - Local cleartext breaches to scan for targets. Uses - multiprocesses, one separate process per file, on - separate worker pool by arguments. Supports file or - folder as input, and filepath globing - -gz LOCAL_GZIP_SRC [LOCAL_GZIP_SRC ...], --gzip LOCAL_GZIP_SRC [LOCAL_GZIP_SRC ...] - Local tar.gz (gzip) compressed breaches to scans for - targets. Uses multiprocesses, one separate process per - file. Supports file or folder as input, and filepath - globing. Looks for 'gz' in filename - -zs LOCAL_ZSTD_SRC [LOCAL_ZSTD_SRC ...], --zstd LOCAL_ZSTD_SRC [LOCAL_ZSTD_SRC ...] - Local zst (zstandard) compressed breaches to scans for - targets. Uses multiprocesses, one separate process per - file. Supports file or folder as input, and filepath - globing. Looks for 'zst' in filename - -sf, --single-file If breach contains big cleartext or tar.gz files, set - this flag to view the progress bar. Disables - concurrent file searching for stability - -ch [CHASE_LIMIT], --chase [CHASE_LIMIT] - Add related emails from hunter.io to ongoing target - list. Define number of emails per target to chase. - Requires hunter.io private API key if used without - power-chase - --power-chase Add related emails from ALL API services to ongoing - target list. Use with --chase - --hide Only shows the first 4 characters of found passwords - to output. Ideal for demonstrations - --debug Print request debug information - --gen-config, -g Generates a configuration file template in the current - working directory & exits. Will overwrite existing - h8mail_config.ini file -``` - ------ - -## :tangerine: Usage examples - -###### Query for a single target - -```bash -$ h8mail -t target@example.com -``` - -###### Query for list of targets, indicate config file for API keys, output to `pwned_targets.csv` -```bash -$ h8mail -t targets.txt -c config.ini -o pwned_targets.csv -``` - -###### Query a list of targets against local copy of the Breach Compilation, pass API keys for [Snusbase](https://snusbase.com/) from the command line -```bash -$ h8mail -t targets.txt -bc ../Downloads/BreachCompilation/ -k "snusbase_url=$snusbase_url,snusbase_token=$snusbase_token" -``` - -###### Query without making API calls against local copy of the Breach Compilation -```bash -$ h8mail -t targets.txt -bc ../Downloads/BreachCompilation/ -sk -``` - -###### Search every .gz file for targets found in targets.txt locally - -```bash -$ h8mail -t targets.txt -gz /tmp/Collection1/ -sk -``` - -###### Check a cleartext dump for target. Add the next 10 related emails to targets to check. Read keys from CLI - -```bash -$ h8mail -t admin@evilcorp.com -lb /tmp/4k_Combo.txt -ch 10 -k "hunterio=ABCDE123" -``` -###### Query username. Read keys from CLI - -```bash -$ h8mail -t JSmith89 -q username -k "dehashed_email=user@email.com" "dehashed_key=ABCDE123" -``` - -###### Query IP. Chase all related targets. Read keys from CLI - - -```bash -$ h8mail -t 42.202.0.42 -q ip -c h8mail_config_priv.ini -ch 2 --power-chase -``` - -###### Fetch URL content (CLI + file). Target all found emails - - -```bash -$ h8mail -u "https://pastebin.com/raw/kQ6WNKqY" "list_of_urls.txt" -``` - - ------ - -## :tangerine: Thanks & Credits - -* [Snusbase](https://snusbase.com/) for being developer friendly -* [kodykinzie](https://twitter.com/kodykinzie) for making a nice [introduction and walkthrough article](https://null-byte.wonderhowto.com/how-to/exploit-recycled-credentials-with-h8mail-break-into-user-accounts-0188600/) and [video](https://www.youtube.com/watch?v=z8G_vBBHtfA) on installing and using h8mail -* [Leak-Lookup](https://leak-lookup.com/) for being developer friendly -* [WeLeakInfo](https://weleakinfo.com/) for being developer friendly -* h8mail's Pypi integration is strongly based on the work of audreyr's [CookieCutter PyPackage](https://github.com/audreyr/cookiecutter-pypackage) -* Logo generated using Hatchful by Shopify -* [Jake Creps](https://twitter.com/jakecreps) for his [h8mail v2 introduction](https://jakecreps.com/2019/06/21/h8mail/) -* [Alejandro Caceres](https://twitter.com/_hyp3ri0n) for making scylla.sh available. Be sure to [support](https://www.buymeacoffee.com/Eiw47ImnT) him if you can -* [IntelX](https://intelx.io) for being developer friendly - -:purple_heart: **h8mail can be found in:** -* [BlackArch Linux](https://blackarch.org/recon.html) -* [Tsurugi DFIR VM](https://tsurugi-linux.org/) -* [Trace Labs OSINT VM](https://www.tracelabs.org/trace-labs-osint-vm/) - - ------ - -## :tangerine: Related open source projects -* [WhatBreach](https://github.com/Ekultek/WhatBreach) by Ekultek -* [HashBuster](https://github.com/s0md3v/Hash-Buster) by s0md3v -* [BaseQuery](https://github.com/g666gle/BaseQuery) by g666gle -* [LeakLooker](https://github.com/woj-ciech/LeakLooker) by woj-ciech -* [buster](https://github.com/sham00n/buster) by sham00n -* [Scavenger](https://github.com/rndinfosecguy/Scavenger) by ndinfosecguy -* [pwndb](https://github.com/davidtavarez/pwndb) by davidtavarez - - ------ - -## :tangerine: Notes - -* Service providers that wish being integrated can send me an email at `k at khast3x dot club` (PGP friendly) -* h8mail is maintained on my free time. Feedback and war stories are welcomed. -* Licence is BSD 3 clause -* My code is [signed](https://help.github.com/en/articles/signing-commits) with my [Keybase](https://keybase.io/ktx) PGP key. You can get it using: -```bash -# curl + gpg pro tip: import ktx's keys -curl https://keybase.io/ktx/pgp_keys.asc | gpg --import - -# the Keybase app can push to gpg keychain, too -keybase pgp pull ktx -``` -___ - -*If you wish to stay updated on this project:* - - -

- -

From 3fcfada7d86cfc0f32e72b83e74a0ce76ea01ec7 Mon Sep 17 00:00:00 2001 From: ktx <1370650+khast3x@users.noreply.github.com> Date: Thu, 28 Jan 2021 10:40:03 +0100 Subject: [PATCH 3/6] Delete requirements.txt --- h8mail/requirements.txt | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 h8mail/requirements.txt diff --git a/h8mail/requirements.txt b/h8mail/requirements.txt deleted file mode 100644 index c047178..0000000 --- a/h8mail/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -zstandard From d3b85b6b9553c91b441a9b84cb0238b7746e8232 Mon Sep 17 00:00:00 2001 From: ktx <1370650+khast3x@users.noreply.github.com> Date: Thu, 28 Jan 2021 10:40:20 +0100 Subject: [PATCH 4/6] Delete classes.py --- h8mail/utils/classes.py | 824 ---------------------------------------- 1 file changed, 824 deletions(-) delete mode 100644 h8mail/utils/classes.py diff --git a/h8mail/utils/classes.py b/h8mail/utils/classes.py deleted file mode 100644 index b6a2ed4..0000000 --- a/h8mail/utils/classes.py +++ /dev/null @@ -1,824 +0,0 @@ -#!/usr/bin/env python -from .intelx import intelx as i -from time import sleep -from .colors import colors as c -import requests -import json -import socket -import sys -import platform -from .version import __version__ - - -class local_breach_target: - """ - Class is called when performing local file search. - This class is meant to store found data, to be later appended to the existing target objects. - local_to_targets() tranforms to target object - Used by cleartext, gzip and zstd search - """ - - def __init__(self, target_data, fp, ln, content): - self.target = target_data - self.filepath = fp - self.line = ln - self.content = content - - def dump(self): - print(f"Email: {self.target}") - print(f"Path: {self.filepath}") - print(f"Line: {self.line}") - print(f"Content: {self.content}") - print() - - -class target: - """ - Main class used to create and follow breach data. - Found data is stored in self.data. Each method increments self.pwned when data is found. - """ - - def __init__(self, target_data, debug=False): - self.headers = { - "User-Agent": "h8mail-v.{h8ver}-OSINT-and-Education-Tool (PythonVersion={pyver}; Platform={platfrm})".format( - h8ver=__version__, - pyver=sys.version.split(" ")[0], - platfrm=platform.platform().split("-")[0], - ) - } - self.target = target_data - self.pwned = 0 - self.data = [()] - self.debug = debug - if debug: - print( - c.fg.red, - f"DEBUG: Created target object for {self.target}", - c.reset, - ) - - def not_exists(self, pattern): - for d in self.data: - if len(d) >= 2: - if d[1] == pattern: - return False - return True - - def make_request( - self, - url, - meth="GET", - timeout=20, - redirs=True, - data=None, - params=None, - verify=True, - auth=None, - ): - try: - response = requests.request( - url=url, - headers=self.headers, - method=meth, - timeout=timeout, - allow_redirects=redirs, - data=data, - params=params, - verify=verify, - auth=auth, - ) - # response = requests.request(url="http://127.0.0.1:8000", headers=self.headers, method=meth, timeout=timeout, allow_redirects=redirs, data=data, params=params) - if self.debug: - c.debug_news("DEBUG: Sent the following---------------------") - print(self.headers) - print(url, meth, data, params) - c.debug_news("DEBUG: Received the following---------------------") - c.debug_news(response.url) - c.debug_news("DEBUG: RESPONSE HEADER---------------------") - print( - "\n".join( - f"{k}: {v}" for k, v in response.headers.items() - ) - ) - c.debug_news("DEBUG: RESPONSE BODY---------------------") - print(response.content) - # print(response) - except Exception as ex: - c.bad_news("Request could not be made for " + self.target) - print(url) - print(ex) - print(response) - return response - - # New HIBP API - def get_hibp3(self, api_key): - try: - c.info_news("[" + self.target + "]>[hibp]") - sleep(1.3) - url = f"https://haveibeenpwned.com/api/v3/breachedaccount/{self.target}" - self.headers.update({"hibp-api-key": api_key}) - response = self.make_request(url) - if response.status_code not in [200, 404]: - c.bad_news("Could not contact HIBP v3 for " + self.target) - print(response.status_code) - return - - if response.status_code == 200: - data = response.json() - for d in data: # Returned type is a dict of Name : Service - for _, ser in d.items(): - self.data.append(("HIBP3", ser)) - self.pwned += 1 - - c.good_news( - "Found {num} breaches for {target} using HIBP v3".format( - num=len(self.data) - 1, target=self.target - ) - ) - self.get_hibp3_pastes() - self.headers.popitem() - elif response.status_code == 404: - c.info_news( - f"No breaches found for {self.target} using HIBP v3" - ) - - else: - c.bad_news( - f"HIBP v3: got API response code {response.status_code} for {self.target}" - ) - - except Exception as e: - c.bad_news("haveibeenpwned v3: " + self.target) - print(e) - - # New HIBP API - def get_hibp3_pastes(self): - try: - c.info_news("[" + self.target + "]>[hibp-paste]") - sleep(1.3) - url = f"https://haveibeenpwned.com/api/v3/pasteaccount/{self.target}" - - response = self.make_request(url) - if response.status_code not in [200, 404]: - c.bad_news("Could not contact HIBP PASTE for " + self.target) - print(response.status_code) - print(response) - return - - if response.status_code == 200: - - data = response.json() - for d in data: # Returned type is a dict of Name : Service - self.pwned += 1 - if "Pastebin" in d["Source"]: - self.data.append( - ("HIBP3_PASTE", "https://pastebin.com/" + d["Id"]) - ) - else: - self.data.append(("HIBP3_PASTE", d["Id"])) - - c.good_news( - "Found {num} pastes for {target} using HIBP v3 Pastes".format( - num=len(data), target=self.target - ) - ) - - elif response.status_code == 404: - c.info_news( - f"No pastes found for {self.target} using HIBP v3 PASTE" - ) - else: - c.bad_news( - f"HIBP v3 PASTE: got API response code {response.status_code} for {self.target}" - ) - - except Exception as ex: - c.bad_news("HIBP v3 PASTE error: " + self.target) - print(ex) - - def get_intelx(self, api_keys): - try: - intel_files = [] - intelx = i(key=api_keys["intelx_key"], ua="h8mail-v.{h8ver}-OSINT-and-Education-Tool (PythonVersion={pyver}; Platform={platfrm})".format( - h8ver=__version__, - pyver=sys.version.split(" ")[0], - platfrm=platform.platform().split("-")[0], - )) - from .intelx_helpers import intelx_getsearch - from .localsearch import local_search - from os import remove, fspath - - maxfile = 10 - if api_keys["intelx_maxfile"]: - maxfile = int(api_keys["intelx_maxfile"]) - search = intelx_getsearch(self.target, intelx, maxfile) - if self.debug: - import json - - print(json.dumps(search, indent=4)) - - for record in search["records"]: - filename = record["systemid"].strip() + ".txt" - intel_files.append(filename) - if record["media"] != 24: - c.info_news( - "Skipping {name}, not text ({type})".format( - type=record["mediah"], name=record["name"] - ) - ) - continue - c.good_news( - "[" - + self.target - + "]>[intelx.io] Fetching " - + record["name"] - + " as file " - + filename - + " (" - + "{:,.0f}".format(record["size"] / float(1 << 20)) - + " MB)" - ) - intelx.FILE_READ(record["systemid"], 0, record["bucket"], filename) - found_list = local_search([filename], [self.target]) - for f in found_list: - self.pwned += 1 - self.data.append( - ( - "INTELX.IO", - "{name} | Line: {line} - {content}".format( - name=record["name"].strip(), - line=f.line, - content=" ".join(f.content.split()), - ), - ) - ) - # print(contents) # Contains search data - for file in intel_files: - if self.debug: - c.info_news( - "[" - + self.target - + f"]>[intelx.io] [DEBUG] Keeping {file}" - ) - else: - c.info_news( - "[" - + self.target - + f"]>[intelx.io] Removing {file}" - ) - remove(file) - - except Exception as ex: - c.bad_news("intelx.io error: " + self.target) - print(ex) - - def get_emailrepio(self, api_key=""): - try: - sleep(0.5) - if len(api_key) != 0: - self.headers.update({"Key": api_key}) - c.info_news("[" + self.target + "]>[emailrep.io+key]") - else: - c.info_news("[" + self.target + "]>[emailrep.io]") - url = f"https://emailrep.io/{self.target}" - response = self.make_request(url) - if response.status_code not in [200, 404, 429]: - c.bad_news("Could not contact emailrep for " + self.target) - print(response.status_code) - print(response) - return - - if response.status_code == 429: - c.info_news( - "[warning] Is your emailrep key working? Get a free API key here: https://bit.ly/3b1e7Pw" - ) - elif response.status_code == 404: - c.info_news( - f"No data found for {self.target} using emailrep.io" - ) - elif response.status_code == 200: - data = response.json() - - self.data.append( - ( - "EMAILREP_INFO", - "Reputation: {rep} | Deliverable: {deli}".format( - rep=data["reputation"].capitalize(), - deli=data["details"]["deliverable"], - ), - ) - ) - - if data["details"]["credentials_leaked"] is True: - self.pwned += int(data["references"]) # or inc num references - if data["references"] == 1: - self.data.append( - ( - "EMAILREP_LEAKS", - f"{data['references']} leaked credential", - ) - ) - else: - self.data.append( - ( - "EMAILREP_LEAKS", - "{} leaked credentials".format(data["references"]), - ) - ) - c.good_news( - "Found {num} breaches for {target} using emailrep.io".format( - num=data["references"], target=self.target - ) - ) - if len(data["details"]["profiles"]) != 0: - for profile in data["details"]["profiles"]: - self.data.append(("EMAILREP_SOCIAL", profile.capitalize())) - c.good_news( - "Found {num} social profiles linked to {target} using emailrep.io".format( - num=len(data["details"]["profiles"]), target=self.target - ) - ) - if "never" in data["details"]["last_seen"]: - return - self.data.append(("EMAILREP_1ST_SN", data["details"]["first_seen"])) - c.good_news( - "{target} was first seen on the {data}".format( - data=data["details"]["first_seen"], target=self.target - ) - ) - self.data.append(("EMAILREP_LASTSN", data["details"]["last_seen"])) - c.good_news( - "{target} was last seen on the {data}".format( - data=data["details"]["last_seen"], target=self.target - ) - ) - else: - c.bad_news( - "emailrep.io: got API response code {code} for {target}".format( - code=response.status_code, target=self.target - ) - ) - if len(api_key) != 0: - self.headers.popitem() - except Exception as ex: - c.bad_news("emailrep.io error: " + self.target) - print(ex) - - def get_scylla(self, user_query="email"): - try: - c.info_news("[" + self.target + "]>[scylla.sh]") - sleep(0.5) - self.headers.update({"Accept": "application/json"}) - if user_query == "email": - uri_scylla = 'Email: "' + self.target + '"' - elif user_query == "password": - uri_scylla = 'Password: "' + self.target + '"' - elif user_query == "username": - uri_scylla = 'User: "' + self.target + '"' - elif user_query == "ip": - uri_scylla = 'IP: "' + self.target + '"' - elif user_query == "hash": - uri_scylla = 'Hash: "' + self.target + '"' - elif user_query == "domain": - uri_scylla = 'Email: "*@' + self.target + '"' - url = "https://scylla.sh/search?q={}".format( - requests.utils.requote_uri(uri_scylla) - ) - - # https://github.com/khast3x/h8mail/issues/64 - response = self.make_request( - url, - verify=False, - auth=requests.auth.HTTPBasicAuth("sammy", "BasicPassword!"), - ) - self.headers.popitem() - - if response.status_code not in [200, 404]: - c.bad_news("Could not contact scylla.sh for " + self.target) - print(response.status_code) - print(response) - return - data = response.json() - total = 0 - for d in data: - for field, k in d["_source"].items(): - if k is not None: - total += 1 - c.good_news( - "Found {num} entries for {target} using scylla.sh ".format( - num=total, target=self.target - ) - ) - for d in data: - for field, k in d["_source"].items(): - if "User" in field and k is not None: - self.data.append(("SCYLLA_USERNAME", k)) - self.pwned += 1 - elif ( - "Email" in field and k is not None and user_query != "email" - ): - self.data.append(("SCYLLA_EMAIL", k)) - self.pwned += 1 - elif "Password" in field and k is not None: - self.data.append(("SCYLLA_PASSWORD", k)) - self.pwned += 1 - elif "PassHash" in field and k is not None: - self.data.append(("SCYLLA_HASH", k)) - self.pwned += 1 - elif "PassSalt" in field and k is not None: - self.data.append(("SCYLLA_HASHSALT", k)) - self.pwned += 1 - elif "IP" in field and k is not None: - self.data.append(("SCYLLA_LASTIP", k)) - self.pwned += 1 - elif "Domain" in field and k is not None: - self.data.append(("SCYLLA_SOURCE", k)) - self.pwned += 1 - except Exception as ex: - c.bad_news("scylla.sh error: " + self.target) - print(ex) - - def get_hunterio_public(self): - try: - c.info_news("[" + self.target + "]>[hunter.io public]") - target_domain = self.target.split("@")[1] - url = f"https://api.hunter.io/v2/email-count?domain={target_domain}" - req = self.make_request(url) - response = req.json() - if response["data"]["total"] != 0: - self.data.append(("HUNTER_PUB", response["data"]["total"])) - c.good_news( - "Found {num} related emails for {target} using hunter.io (public)".format( - num=response["data"]["total"], target=self.target - ) - ) - except Exception as ex: - c.bad_news("hunter.io (public API) error: " + self.target) - print(ex) - - def get_hunterio_private(self, api_key): - try: - c.info_news("[" + self.target + "]>[hunter.io private]") - target_domain = self.target.split("@")[1] - url = f"https://api.hunter.io/v2/domain-search?domain={target_domain}&api_key={api_key}" - req = self.make_request(url) - response = req.json() - b_counter = 0 - for e in response["data"]["emails"]: - self.data.append(("HUNTER_RELATED", e["value"])) - b_counter += 1 - if self.pwned != 0: - self.pwned += 1 - c.good_news( - "Found {num} related emails for {target} using hunter.io (private)".format( - num=b_counter, target=self.target - ) - ) - except Exception as ex: - c.bad_news( - f"hunter.io (private API) error for {self.target}:" - ) - print(ex) - - def get_snusbase(self, api_url, api_key, user_query): - try: - if user_query == "ip": - user_query = "lastip" - if user_query in ["domain"]: - c.bad_news( - f"Snusbase does not support {user_query} search (yet)" - ) - return - c.info_news("[" + self.target + "]>[snusbase]") - url = api_url - self.headers.update({"Authorization": api_key}) - payload = {"type": user_query, "term": self.target} - req = self.make_request(url, meth="POST", data=payload) - self.headers.popitem() - response = req.json() - c.good_news( - "Found {num} entries for {target} using Snusbase".format( - num=len(response["result"]), target=self.target - ) - ) - for result in response["result"]: - if result["email"] and self.not_exists(result["email"]): - self.data.append(("SNUS_RELATED", result["email"].strip())) - if result["username"]: - self.data.append(("SNUS_USERNAME", result["username"])) - self.pwned += 1 - if result["password"]: - self.data.append(("SNUS_PASSWORD", result["password"])) - self.pwned += 1 - if result["hash"]: - if result["salt"]: - self.data.append( - ( - "SNUS_HASH_SALT", - result["hash"].strip() + " : " + result["salt"].strip(), - ) - ) - self.pwned += 1 - else: - self.data.append(("SNUS_HASH", result["hash"])) - self.pwned += 1 - if result["lastip"]: - self.data.append(("SNUS_LASTIP", result["lastip"])) - self.pwned += 1 - if result["name"]: - self.data.append(("SNUS_NAME", result["name"])) - self.pwned += 1 - if result["tablenr"] and self.not_exists(result["tablenr"]): - self.data.append(("SNUS_SOURCE", result["tablenr"])) - - except Exception as ex: - c.bad_news(f"Snusbase error with {self.target}") - print(ex) - - def get_leaklookup_pub(self, api_key): - try: - c.info_news("[" + self.target + "]>[leaklookup public]") - url = "https://leak-lookup.com/api/search" - payload = {"key": api_key, "type": "email_address", "query": self.target} - req = self.make_request(url, meth="POST", data=payload, timeout=20) - response = req.json() - if "false" in response["error"] and len(response["message"]) != 0: - c.good_news( - "Found {num} entries for {target} using LeakLookup (public)".format( - num=len(response["message"]), target=self.target - ) - ) - for result in response["message"]: - self.pwned += 1 - self.data.append(("LEAKLOOKUP_PUB", result)) - if "false" in response["error"] and len(response["message"]) == 0: - c.info_news( - f"No breaches found for {self.target} using Leak-lookup (public)" - ) - - except Exception as ex: - c.bad_news( - f"Leak-lookup error with {self.target} (public)" - ) - print(ex) - - def get_leaklookup_priv(self, api_key, user_query): - try: - if user_query == "ip": - user_query = "ipadress" - if user_query in ["hash"]: - c.bad_news( - f"Leaklookup does not support {user_query} search (yet)" - ) - return - c.info_news("[" + self.target + "]>[leaklookup private]") - url = "https://leak-lookup.com/api/search" - payload = {"key": api_key, "type": user_query, "query": self.target} - req = self.make_request(url, meth="POST", data=payload, timeout=60) - response = req.json() - if "false" in response["error"] and len(response["message"]) != 0: - b_counter = 0 - for db, data in response["message"].items(): - for d in data: - if "username" in d.keys(): - self.pwned += 1 - self.data.append(("LKLP_USERNAME", d["username"])) - if "email_address" in d.keys() and self.not_exists( - d["email_address"] - ): - self.data.append( - ("LKLP_RELATED", d["email_address"].strip()) - ) - if "password" in d.keys(): - self.pwned += 1 - self.data.append(("LKLP_PASSWORD", d["password"])) - b_counter += 1 - if "hash" in d.keys(): - self.pwned += 1 - self.data.append(("LKLP_HASH", d["password"])) - b_counter += 1 - if "ipaddress" in d.keys(): - self.pwned += 1 - self.data.append(("LKLP_LASTIP", d["ipaddress"])) - for tag in [ - "address", - "address1", - "address2", - "country", - "zip", - "zipcode", - "postcode", - "state", - ]: - if tag in d.keys(): - self.pwned += 1 - self.data.append( - ("LKLP_GEO", d[tag] + " (type: " + tag + ")") - ) - for tag in [ - "firstname", - "middlename", - "lastname", - "mobile", - "number", - "userid", - ]: - if tag in d.keys(): - self.pwned += 1 - self.data.append( - ("LKLP_ID", d[tag] + " (type: " + tag + ")") - ) - if self.not_exists(db): - self.data.append(("LKLP_SOURCE", db)) - - c.good_news( - "Found {num} entries for {target} using LeakLookup (private)".format( - num=b_counter, target=self.target - ) - ) - - if "false" in response["error"] and len(response["message"]) == 0: - c.info_news( - f"No breaches found for {self.target} using Leak-lookup (private)" - ) - except Exception as ex: - c.bad_news( - f"Leak-lookup error with {self.target} (private)" - ) - print(ex) - - def get_weleakinfo_priv(self, api_key, user_query): - try: - c.info_news("[" + self.target + "]>[weleakinfo priv]") - sleep(0.4) - url = "https://api.weleakinfo.com/v3/search" - self.headers.update({"Authorization": "Bearer " + api_key}) - self.headers.update({"Content-Type": "application/x-www-form-urlencoded"}) - - payload = {"type": user_query, "query": self.target} - req = self.make_request(url, meth="POST", data=payload, timeout=30) - self.headers.popitem() - self.headers.popitem() - response = req.json() - if req.status_code == 400: - c.bad_news( - f"Got WLI API response code {req.status_code}: Invalid search type provided" - ) - return - elif req.status_code != 200: - c.bad_news(f"Got WLI API response code {req.status_code} (private)") - return - if req.status_code == 200: - if response["Success"] is False: - c.bad_news( - f"WeLeakInfo (private) error response {response['Message']}" - ) - return - c.good_news( - "Found {num} entries for {target} using WeLeakInfo (private)".format( - num=response["Total"], target=self.target - ) - ) - self.data.append(("WLI_TOTAL", response["Total"])) - if response["Total"] == 0: - return - for result in response["Data"]: - if "Username" in result: - self.data.append(("WLI_USERNAME", result["Username"])) - if "Email" in result and self.not_exists(result["Email"]): - self.data.append(("WLI_RELATED", result["Email"].strip())) - if "Password" in result: - self.data.append(("WLI_PASSWORD", result["Password"])) - self.pwned += 1 - if "Hash" in result: - self.data.append(("WLI_HASH", result["Hash"])) - self.pwned += 1 - if "Database" in result and self.not_exists(result["Database"]): - self.data.append(("WLI_SOURCE", result["Database"])) - except Exception as ex: - c.bad_news( - f"WeLeakInfo error with {self.target} (private)" - ) - print(ex) - - def get_weleakinfo_pub(self, api_key): - try: - c.info_news("[" + self.target + "]>[weleakinfo public]") - url = "https://api.weleakinfo.com/v3/public/email/{query}".format( - query=self.target - ) - self.headers.update({"Authorization": "Bearer " + api_key}) - req = self.make_request(url, timeout=30) - self.headers.popitem() - response = req.json() - if req.status_code != 200: - c.bad_news(f"Got WLI API response code {req.status_code} (public)") - return - else: - c.good_news( - "Found {num} entries for {target} using WeLeakInfo (public)".format( - num=response["Total"], target=self.target - ) - ) - if response["Success"] is False: - c.bad_news(response["Message"]) - return - self.data.append(("WLI_PUB_TOTAL", response["Total"])) - if response["Total"] == 0: - return - for name, data in response["Data"].items(): - self.data.append(("WLI_PUB_SRC", name + " (" + str(data) + ")")) - except Exception as ex: - c.bad_news( - f"WeLeakInfo error with {self.target} (public)" - ) - print(ex) - - def get_dehashed(self, api_email, api_key, user_query): - try: - # New Dehashed API needs fixing, waiting for devs to respond - c.bad_news("Dehashed is temporarily unavailable") - c.bad_news("This should be fixed in the next updates\n") - return - - if user_query == "hash": - user_query == "hashed_password" - if user_query == "ip": - user_query == "ip_address" - - c.info_news("[" + self.target + "]>[dehashed]") - url = "https://api.dehashed.com/search?query=" - if user_query == "domain": - search_query = "email" + ":" + '"*@' + self.target + '"' - else: - search_query = user_query + ":" + '"' + self.target + '"' - self.headers.update({"Accept": "application/json"}) - req = self.make_request( - url + search_query, meth="GET", timeout=60, auth=(api_email, api_key) - ) - if req.status_code == 200: - response = req.json() - if response["total"] is not None: - c.good_news( - "Found {num} entries for {target} using Dehashed.com".format( - num=str(response["total"]), target=self.target - ) - ) - - for result in response["entries"]: - if ( - "username" in result - and result["username"] is not None - and len(result["username"].strip()) > 0 - ): - self.data.append(("DHASHD_USERNAME", result["username"])) - if ( - "email" in result - and self.not_exists(result["email"]) - and result["email"] is not None - and len(result["email"].strip()) > 0 - ): - self.data.append(("DHASHD_RELATED", result["email"].strip())) - if ( - "password" in result - and result["password"] is not None - and len(result["password"].strip()) > 0 - ): - self.data.append(("DHASHD_PASSWORD", result["password"])) - self.pwned += 1 - if ( - "hashed_password" in result - and result["hashed_password"] is not None - and len(result["hashed_password"].strip()) > 0 - ): - self.data.append(("DHASHD_HASH", result["hashed_password"])) - self.pwned += 1 - for tag in ["name", "vin", "address", "phone"]: - if ( - tag in result - and result[tag] is not None - and len(result[tag].strip()) > 0 - ): - self.data.append( - ("DHASHD_ID", result[tag] + " (type: " + tag + ")") - ) - self.pwned += 1 - - if "obtained_from" in result and self.not_exists( - result["obtained_from"] - ): - self.data.append(("DHASHD_SOURCE", result["obtained_from"])) - - if response["balance"] is not None: - self.data.append( - ( - "DHASHD_CREDITS", - str(response["balance"]) + " DEHASHED CREDITS REMAINING", - ) - ) - else: - c.bad_news("Dehashed error: status code " + req.status_code) - self.headers.popitem() - except Exception as ex: - c.bad_news(f"Dehashed error with {self.target}") - print(ex) From 7df35042a7179eb886de3071f6e2f79d4fb78867 Mon Sep 17 00:00:00 2001 From: ktx <1370650+khast3x@users.noreply.github.com> Date: Thu, 28 Jan 2021 10:40:38 +0100 Subject: [PATCH 5/6] Delete run.py --- h8mail/utils/run.py | 381 -------------------------------------------- 1 file changed, 381 deletions(-) delete mode 100644 h8mail/utils/run.py diff --git a/h8mail/utils/run.py b/h8mail/utils/run.py deleted file mode 100644 index 9dbe8fd..0000000 --- a/h8mail/utils/run.py +++ /dev/null @@ -1,381 +0,0 @@ -# -*- coding: utf-8 -*- -# Most imports are after python2/3 check further down -import configparser -import argparse -import os -import re -import time -import sys - -from .breachcompilation import breachcomp_check -from .classes import target -from .colors import colors as c -from .helpers import ( - fetch_emails, - find_files, - get_config_from_file, - get_emails_from_file, - print_banner, - save_results_csv, - check_latest_version, - check_scylla_online, -) -from .localsearch import local_search, local_search_single, local_to_targets -from .localgzipsearch import local_gzip_search, local_search_single_gzip -from .localzstdsearch import local_zstd_search, local_search_single_zstd -from .summary import print_summary -from .chase import chase -from .print_results import print_results -from .gen_config import gen_config_file -from .url import target_urls - - -def target_factory(targets, user_args): - """ - Receives list of emails and user args. Fetchs API keys from config file using user_args path and cli keys. - For each target, launch target.methods() associated to found config artifacts. - Handles chase logic with counters from enumerate() - """ - # Removing duplicates here to avoid dups from chasing - targets = list(set(targets)) - - finished = [] - if user_args.config_file is not None or user_args.cli_apikeys is not None: - api_keys = get_config_from_file(user_args) - else: - api_keys = None - init_targets_len = len(targets) - - query = "email" - skip_default_queries = False - if user_args.user_query is not None: - query = user_args.user_query - skip_default_queries = True # custom query skips default query automatically - - scylla_up = False - if user_args.skip_defaults is False: - scylla_up = check_scylla_online() - - - - for counter, t in enumerate(targets): - c.info_news("Target factory started for {target}".format(target=t)) - if user_args.debug: - current_target = target(t, debug=True) - else: - current_target = target(t) - if not skip_default_queries: - if not user_args.skip_defaults: - current_target.get_hunterio_public() - ## emailrep seems to insta-block h8mail user agent without a key - # if api_keys is None or "emailrep" not in api_keys: - # current_target.get_emailrepio() - # elif ( - # api_keys is not None and "emailrep" in api_keys and query == "email" - # ): - # current_target.get_emailrepio(api_keys["emailrep"]) - - if api_keys is not None: - if "hibp" in api_keys and query == "email": - current_target.get_hibp3(api_keys["hibp"]) - if "emailrep" in api_keys and query == "email": - current_target.get_emailrepio(api_keys["emailrep"]) - if "hunterio" in api_keys and query == "email": - current_target.get_hunterio_private(api_keys["hunterio"]) - if "intelx_key" in api_keys: - current_target.get_intelx(api_keys) - if "snusbase_token" in api_keys: - if "snusbase_url" in api_keys: - snusbase_url = api_keys["snusbase_url"] - else: - snusbase_url = "http://api.snusbase.com/v2/search" - current_target.get_snusbase( - snusbase_url, api_keys["snusbase_token"], query - ) - if "leak-lookup_priv" in api_keys: - current_target.get_leaklookup_priv(api_keys["leak-lookup_priv"], query) - if "leak-lookup_pub" in api_keys and query == "email": - current_target.get_leaklookup_pub(api_keys["leak-lookup_pub"]) - if "weleakinfo_pub" in api_keys and query == "email": - current_target.get_weleakinfo_pub(api_keys["weleakinfo_pub"]) - if "weleakinfo_priv" in api_keys: - current_target.get_weleakinfo_priv(api_keys["weleakinfo_priv"], query) - if "dehashed_key" in api_keys: - if "dehashed_email" in api_keys: - current_target.get_dehashed( - api_keys["dehashed_email"], api_keys["dehashed_key"], query - ) - else: - c.bad_news("Missing Dehashed email") - if scylla_up: - current_target.get_scylla(query) - - # Chasing - if user_args.chase_limit and counter <= init_targets_len: - user_args_force_email = user_args - user_args_force_email.user_query = "email" - user_args_force_email.chase_limit -= 1 - finished_chased = target_factory( - chase(current_target, user_args), user_args_force_email - ) - finished.extend((finished_chased)) - finished.append(current_target) - return finished - - -def h8mail(user_args): - """ - Handles most user arg logic. Creates a list() of targets from user input. - Starts the target object factory loop; starts local searches after factory if in user inputs - Prints results, saves to csv if in user inputs - """ - - if user_args.user_targets and user_args.user_urls: - c.bad_news("Cannot use --url with --target. Use one or the other.") - exit(1) - - if not user_args.user_targets and not user_args.user_urls: - c.bad_news("Missing Target or URL") - exit(1) - - start_time = time.time() - - import warnings - - warnings.filterwarnings('ignore', message='Unverified HTTPS request') - - targets = [] - if user_args.user_urls: - targets = target_urls(user_args) - if len(targets) == 0: - c.bad_news("No targets found in URLs. Quitting") - exit(0) - - # If we found emails from URLs, `targets` array already has stuff - if len(targets) != 0: - if user_args.user_targets is None: - user_args.user_targets = [] - user_args.user_targets.extend(targets) - - else: # Find targets in user input or file - if user_args.user_targets is not None: - for arg in user_args.user_targets: - user_stdin_target = fetch_emails(arg, user_args) - if os.path.isfile(arg): - c.info_news("Reading from file " + arg) - targets.extend(get_emails_from_file(arg, user_args)) - elif user_stdin_target: - targets.extend(user_stdin_target) - else: - c.bad_news("No targets found in user input. Quitting") - exit(0) - - c.info_news("Removing duplicates") - targets = list(set(targets)) - - c.good_news("Targets:") - for t in targets: - c.good_news(t) - - # Launch - breached_targets = target_factory(targets, user_args) - - # These are not done inside the factory as the factory iterates over each target individually - # The following functions perform line by line checks of all targets per line - - if user_args.bc_path: - breached_targets = breachcomp_check(breached_targets, user_args.bc_path) - - local_found = None - # Handle cleartext search - if user_args.local_breach_src: - for arg in user_args.local_breach_src: - res = find_files(arg) - if user_args.single_file: - local_found = local_search_single(res, targets) - else: - local_found = local_search(res, targets) - if local_found is not None: - breached_targets = local_to_targets( - breached_targets, local_found, user_args - ) - # Handle gzip search - if user_args.local_gzip_src: - for arg in user_args.local_gzip_src: - res = find_files(arg, "gz") - if user_args.single_file: - local_found = local_search_single_gzip(res, targets) - else: - local_found = local_gzip_search(res, targets) - if local_found is not None: - breached_targets = local_to_targets( - breached_targets, local_found, user_args - ) - # Handle zstd search - if user_args.local_zstd_src: - for arg in user_args.local_zstd_src: - res = find_files(arg, "zst") - if user_args.single_file: - local_found = local_search_single_zstd(res, targets) - else: - local_found = local_zstd_search(res, targets) - if local_found is not None: - breached_targets = local_to_targets( - breached_targets, local_found, user_args - ) - - print_results(breached_targets, user_args.hide) - - print_summary(start_time, breached_targets) - if user_args.output_file: - save_results_csv(user_args.output_file, breached_targets) - - -def parse_args(args): - """ - Seperate functions to make it easier to run tests - Pass args as an array - """ - parser = argparse.ArgumentParser( - description="Email information and password lookup tool", prog="h8mail" - ) - - parser.add_argument( - "-t", - "--targets", - dest="user_targets", - help="Either string inputs or files. Supports email pattern matching from input or file, filepath globing and multiple arguments", - nargs="+", - ) - parser.add_argument( - "-u", - "--url", - dest="user_urls", - help="Either string inputs or files. Supports URL pattern matching from input or file, filepath globing and multiple arguments. Parse URLs page for emails. Requires http:// or https:// in URL.", - nargs="+", - ) - parser.add_argument( - "-q", - "--custom-query", - dest="user_query", - help='Perform a custom query. Supports username, password, ip, hash, domain. Performs an implicit "loose" search when searching locally', - ) - parser.add_argument( - "--loose", - dest="loose", - help="Allow loose search by disabling email pattern recognition. Use spaces as pattern seperators", - action="store_true", - default=False, - ) - parser.add_argument( - "-c", - "--config", - dest="config_file", - help="Configuration file for API keys. Accepts keys from Snusbase, WeLeakInfo, Leak-Lookup, HaveIBeenPwned, Emailrep, Dehashed and hunterio", - nargs="+", - ) - parser.add_argument( - "-o", "--output", dest="output_file", help="File to write CSV output" - ) - parser.add_argument( - "-bc", - "--breachcomp", - dest="bc_path", - help="Path to the breachcompilation torrent folder. Uses the query.sh script included in the torrent", - ) - parser.add_argument( - "-sk", - "--skip-defaults", - dest="skip_defaults", - help="Skips HaveIBeenPwned and HunterIO check. Ideal for local scans", - action="store_true", - default=False, - ) - parser.add_argument( - "-k", - "--apikey", - dest="cli_apikeys", - help='Pass config options. Supported format: "K=V,K=V"', - nargs="+", - ) - parser.add_argument( - "-lb", - "--local-breach", - dest="local_breach_src", - help="Local cleartext breaches to scan for targets. Uses multiprocesses, one separate process per file, on separate worker pool by arguments. Supports file or folder as input, and filepath globing", - nargs="+", - ) - parser.add_argument( - "-gz", - "--gzip", - dest="local_gzip_src", - help="Local tar.gz (gzip) compressed breaches to scans for targets. Uses multiprocesses, one separate process per file. Supports file or folder as input, and filepath globing. Looks for 'gz' in filename", - nargs="+", - ) - parser.add_argument( - "-zs", - "--zstd", - dest="local_zstd_src", - help="Local zst (zstandard) compressed breaches to scans for targets. Uses multiprocesses, one separate process per file. Supports file or folder as input, and filepath globing. Looks for 'zst' in filename", - nargs="+", - ) - parser.add_argument( - "-sf", - "--single-file", - dest="single_file", - help="If breach contains big cleartext or tar.gz files, set this flag to view the progress bar. Disables concurrent file searching for stability", - action="store_true", - default=False, - ), - parser.add_argument( - "-ch", - "--chase", - dest="chase_limit", - help="Add related emails from hunter.io to ongoing target list. Define number of emails per target to chase. Requires hunter.io private API key if used without power-chase", - type=int, - nargs="?", - ), - parser.add_argument( - "--power-chase", - dest="power_chase", - help="Add related emails from ALL API services to ongoing target list. Use with --chase", - action="store_true", - default=False, - ), - parser.add_argument( - "--hide", - dest="hide", - help="Only shows the first 4 characters of found passwords to output. Ideal for demonstrations", - action="store_true", - default=False, - ), - parser.add_argument( - "--debug", - dest="debug", - help="Print request debug information", - action="store_true", - default=False, - ), - parser.add_argument( - "--gen-config", - "-g", - dest="gen_config", - help="Generates a configuration file template in the current working directory & exits. Will overwrite existing h8mail_config.ini file", - action="store_true", - default=False, - ), - - return parser.parse_args(args) - - -def main(): - - print_banner("warn") - print_banner("version") - print_banner() - check_latest_version() - user_args = parse_args(sys.argv[1:]) - if user_args.gen_config: - gen_config_file() - exit(0) - h8mail(user_args) From 47e4bd0706d89f97841f9b16ceb4561c4acd1f77 Mon Sep 17 00:00:00 2001 From: ktx <1370650+khast3x@users.noreply.github.com> Date: Thu, 28 Jan 2021 10:41:22 +0100 Subject: [PATCH 6/6] Delete setup.py --- setup.py | 51 --------------------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index a9077b6..0000000 --- a/setup.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -"""The setup script.""" -from setuptools import setup, find_packages, Extension -from distutils.util import convert_path -import os - -here = os.path.abspath(os.path.dirname(__file__)) -exec(open(os.path.join(here, "h8mail/utils/version.py")).read()) - -with open("PyPi.rst") as readme_file: - readme = readme_file.read() - -# with open("HISTORY.rst") as history_file: -# history = history_file.read() - -requirements = ["requests", "zstandard"] - -setup_requirements = ["requests", "zstandard"] - -test_requirements = ["requests", "zstandard"] - -setup( - author="khast3x", - author_email="k@khast3x.club", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Natural Language :: English", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Environment :: Console", - ], - description="Email OSINT and password breach hunting. Use h8mail to find passwords through different breach and reconnaissance services, or using your local data", - install_requires=requirements, - license="BSD license", - # long_description_content_type="text/markdown", - long_description=readme + "\n\n", - # long_description=readme + "\n\n" + history, - include_package_data=True, - keywords="h8mail", - name="h8mail", - packages=find_packages(), - entry_points={"console_scripts": ["h8mail = h8mail.__main__:main"]}, - setup_requires=setup_requirements, - url="https://github.com/khast3x/h8mail", - version=__version__, - zip_safe=False, -)