Merge pull request #1 from amsprost/development

Development

.editorconfig

@@ -0,0 +1,12 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.php]
+indent_size = 4

.env

@@ -19,3 +19,5 @@ APP_SECRET=c7bb60cab041aecda02f06effa4c969b
 #TRUSTED_PROXIES=127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
 #TRUSTED_HOSTS='^(localhost|example\.com)$'
 ###< symfony/framework-bundle ###
+
+APP_THREADS_NUMBER=5

.env.test

@@ -0,0 +1,4 @@
+# define your env variables for the test env here
+KERNEL_CLASS='App\Kernel'
+APP_SECRET='$ecretf0rt3st'
+SYMFONY_DEPRECATIONS_HELPER=999999

.gitignore

@@ -9,3 +9,20 @@
 /vendor/
 /.idea
 ###< symfony/framework-bundle ###
+
+###> phpunit/phpunit ###
+/phpunit.xml
+.phpunit.result.cache
+###< phpunit/phpunit ###
+
+###> symfony/phpunit-bridge ###
+.phpunit
+.phpunit.result.cache
+/phpunit.xml
+###< symfony/phpunit-bridge ###
+
+output/*
+
+!.gitkeep
+
+node_modules

.hooks/pre-push.sample

@@ -0,0 +1 @@
+composer run qa-checks

.versionrc

@@ -0,0 +1,36 @@
+{
+  "types": [
+    {
+      "type": "feat",
+      "section": "Features"
+    },
+    {
+      "type": "fix",
+      "section": "Bug Fixes"
+    },
+    {
+      "type": "docs",
+      "section": "Documentation"
+    },
+    {
+      "type": "test",
+      "section": "Tests"
+    },
+    {
+      "type": "build",
+      "section": "Build System"
+    },
+    {
+      "type": "ci"
+    },
+    {
+      "type": "chore",
+      "section": "Chores"
+    },
+    {
+      "type": "deps",
+      "section": "Dependencies"
+    }
+  ],
+  "issueUrlFormat": "https://team.linkorb.com/cards/{{id}}"
+}

Dockerfile

@@ -1,15 +1,15 @@
 FROM php:7.4-zts
 USER root
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends git zip unzip libzip4 libzip-dev ssh libxml2-dev && \
+    apt-get install -y --no-install-recommends git zip unzip libzip4 libzip-dev ssh libxml2-dev sqlite3 libsqlite3-dev && \
     curl -sSL https://getcomposer.org/composer.phar -o /usr/bin/composer && \
     chmod +x /usr/bin/composer && \
     composer selfupdate && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
     mkdir /app
 
-RUN docker-php-ext-install opcache && docker-php-ext-enable opcache
+RUN docker-php-ext-install opcache mysqli pdo pdo_mysql && docker-php-ext-enable opcache mysqli pdo pdo_mysql
 
 RUN curl -sSL https://github.com/krakjoe/parallel/archive/develop.zip -o /tmp/parallel.zip \
     && unzip /tmp/parallel.zip -d /tmp \
@@ -20,6 +20,11 @@ RUN curl -sSL https://github.com/krakjoe/parallel/archive/develop.zip -o /tmp/pa
     && make install \
     && rm -rf /tmp/parallel*
 
+RUN docker-php-ext-enable parallel
+
+RUN mkdir /db
+RUN /usr/bin/sqlite3 /db/test.db
+
 RUN composer global require hirak/prestissimo
 
 WORKDIR /app

README.md

@@ -0,0 +1,35 @@
+Table Archiver
+==============
+This is a CLI tool to export old data from any SQL database table into ranged ndjson files 
+that you can then wherever you like.
+
+## Installation
+
+    git clone [email protected]:linkorb/table-archiver.git
+    cd table-archiver
+    docker-compose up -d
+    docker-compose exec app composer install # install PHP dependencies
+
+## Usage
+Command looks like:
+
+    ./bin/console linkorb:table:archive {db_dsn} {table_name} {mode} {date_column} [max_stamp]\
+See `./bin/console linkorb:table:archive --help` for more info
+
+(To run it from docker container you need to prepend it with `docker-compose exec app `...)
+
+All gzipped [ndjson](http://ndjson.org/) files can be found under `./output` directory
+
+If you want to change threads number you can do that easily by changing `APP_THREADS_NUMBER` in `.env`
+
+With big date ranges it's recommended to run command with disabled (or increased) memory limit 
+(prepended `php -d memory_limit=-1`). If that's not an option you can bypass caching for writers by passing `no-cache`
+option. Be aware that this setting noticeable affects performance.
+
+### Example:
+
+    ./bin/console linkorb:table:archive mysql://root:[email protected]:3306/test target_table YEAR_MONTH_DAY timestamp 20200101
+
+## Git hooks
+
+There are some git hooks under `.hooks` directory. Feel free to copy & adjust & use them

bin/phpunit

@@ -0,0 +1,13 @@
+#!/usr/bin/env php
+<?php
+
+if (!file_exists(dirname(__DIR__).'/vendor/symfony/phpunit-bridge/bin/simple-phpunit.php')) {
+    echo "Unable to find the `simple-phpunit.php` script in `vendor/symfony/phpunit-bridge/bin/`.\n";
+    exit(1);
+}
+
+if (false === getenv('SYMFONY_PHPUNIT_DIR')) {
+    putenv('SYMFONY_PHPUNIT_DIR='.__DIR__.'/.phpunit');
+}
+
+require dirname(__DIR__).'/vendor/symfony/phpunit-bridge/bin/simple-phpunit.php';

commitlint.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+    extends: ['@commitlint/config-conventional']
+};

composer.json

@@ -5,13 +5,26 @@
         "php": ">=7.4",
         "ext-ctype": "*",
         "ext-iconv": "*",
+        "ext-json": "*",
+        "ext-parallel": "*",
+        "ext-pdo": "*",
+        "linkorb/connector": "^1.2",
         "symfony/console": "5.1.*",
         "symfony/dotenv": "5.1.*",
         "symfony/flex": "^1.3.1",
         "symfony/framework-bundle": "5.1.*",
-        "symfony/yaml": "5.1.*"
+        "symfony/yaml": "5.1.*",
+        "ext-zlib": "*"
     },
     "require-dev": {
+        "phpstan/phpstan": "^0.12.51",
+        "phpstan/phpstan-symfony": "^0.12.10",
+        "phpunit/phpunit": "^9",
+        "sebastian/phpcpd": "^6.0",
+        "sensiolabs/security-checker": "^6.0",
+        "squizlabs/php_codesniffer": "^3.5",
+        "symfony/phpunit-bridge": "^5.1",
+        "wapmorgan/php-code-fixer": "^2.0"
     },
     "config": {
         "optimize-autoloader": true,
@@ -41,14 +54,29 @@
     },
     "scripts": {
         "auto-scripts": {
-            "cache:clear": "symfony-cmd"
+            "cache:clear": "symfony-cmd",
+            "security-checker security:check": "script"
         },
         "post-install-cmd": [
             "@auto-scripts"
         ],
         "post-update-cmd": [
             "@auto-scripts"
-        ]
+        ],
+        "qa-checks": [
+            "@phpstan",
+            "@phpcs",
+            "@phpcpd",
+            "@phpcf",
+            "@security-check",
+            "@phpunit"
+        ],
+        "phpstan": "./vendor/bin/phpstan analyze --level=5 ./src/",
+        "phpcs": "./vendor/bin/phpcs ./src/",
+        "phpcpd": "./vendor/bin/phpcpd --fuzzy ./src/",
+        "phpcf": "./vendor/bin/phpcf --target 7.1 ./src/",
+        "phpunit": "./vendor/bin/phpunit",
+        "security-check": "./vendor/bin/security-checker security:check ./composer.lock"
     },
     "conflict": {
         "symfony/symfony": "*"