diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..8ab1274 --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,9 @@ +Changelog +========= + +0.3.0 +----- + +- Multithreading support. + + The ``ReaderThreads`` option in the plugin configuration file sets the number of threads used to process pmacct's output. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..7c81086 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include LICENSE +include README.rst +include CHANGES.rst +include MANIFEST.in diff --git a/README.md b/README.rst similarity index 83% rename from README.md rename to README.rst index 3ecacbc..04565a1 100644 --- a/README.md +++ b/README.rst @@ -12,29 +12,36 @@ pmacct-to-elasticsearch Optionally, some **data transformations** can be configured, to allow pmacct-to-elasticsearch to **add or remove fields** to/from the output documents that are sent to ElasticSearch for indexing. These additional fields may be useful to enhance graphs and reports legibility, or to add a further level of aggregation or filtering. -## Installation +Installation +------------ -Clone the repository and run the ./install script: +Install the program using pip: + + pip install pmacct-to-elasticsearch + +Then clone the repository and run the ./install script to setup your system: # cd /usr/local/src/ # git clone https://github.com/pierky/pmacct-to-elasticsearch.git # cd pmacct-to-elasticsearch/ # ./install -## Configuration +Configuration +------------- Please refer to the [CONFIGURATION.md](CONFIGURATION.md) file. The [TRANSFORMATIONS.md](TRANSFORMATIONS.md) file contains details about data transformations configuration. A simple tutorial on pmacct integration with ElasticSearch/Kibana using pmacct-to-elasticsearch can be found at http://blog.pierky.com/integration-of-pmacct-with-elasticsearch-and-kibana. -## Future work +Future work +----------- -- Improve performances by introducing multi-thread processing of input data. - Add support of more pmacct output formats (CSV, Apache Avro, ...). - Read input from stdin pipes too. -## Author +Author +------ -Pier Carlo Chiodi - http://pierky.com/aboutme +Pier Carlo Chiodi - https://pierky.com/ -Blog: http://blog.pierky.com Twitter: [@pierky](http://twitter.com/pierky) +Blog: https://blog.pierky.com/ Twitter: `@pierky `_ diff --git a/install b/install index dd322c1..6a59c28 100755 --- a/install +++ b/install @@ -3,9 +3,21 @@ # if you change the following CONF_DIR value you must # change it in pmacct-to-elasticsearch too. CONF_DIR=/etc/p2es - BIN_DIR=/usr/local/bin +if [ ! -e "$BIN_DIR/pmacct-to-elasticsearch" ]; then + echo "ERROR: pmacct-to-elasticsearch not found in $BIN_DIR." + echo "" + echo "If it is not installed, please install it using " + echo "" + echo " pip install pmacct-to-elasticsearch" + echo "" + echo "otherwise please consider editing this installation script" + echo "to set the BIN_DIR variable to the directory where " + echo "pmacct-to-elasticsearch is currently installed." + exit 1 +fi + FORCE="no" while getopts "F" opt; do @@ -17,9 +29,8 @@ done echo "" echo "pmacct-to-elasticsearch installation script" echo "" -echo "The following directories will be used for the installation:" +echo "The following directory will be used for the installation:" echo " - $CONF_DIR: configuration files" -echo " - $BIN_DIR: the main pmacct-to-elasticsearch program" echo "" echo "With -F option, Replace all files without confirmation" echo "" @@ -110,20 +121,6 @@ fi # ----------------------------------------------------------- -echo -n "Copying pmacct-to-elasticsearch program to $BIN_DIR... " - -cp pmacct-to-elasticsearch $BIN_DIR/ - -if [ $? -ne 0 ]; then - echo "ERROR - exit code $?" - exit 1 -else - echo "OK" - chmod u+x $BIN_DIR/pmacct-to-elasticsearch -fi - -# ----------------------------------------------------------- - echo -n "Copying crontab job fragment to /etc/cron.d... " if [ -d /etc/cron.d ]; then diff --git a/pierky/p2es/version.py b/pierky/p2es/version.py new file mode 100644 index 0000000..493f741 --- /dev/null +++ b/pierky/p2es/version.py @@ -0,0 +1 @@ +__version__ = "0.3.0" diff --git a/scripts/pmacct-to-elasticsearch b/scripts/pmacct-to-elasticsearch index 9a2a777..48a4be9 100755 --- a/scripts/pmacct-to-elasticsearch +++ b/scripts/pmacct-to-elasticsearch @@ -17,10 +17,11 @@ import sys from pierky.p2es.es import * from pierky.p2es.transformations import * +from pierky.p2es.version import __version__ from pierky.p2es.workers import * APP_NAME = 'pmacct-to-elasticsearch' -CURRENT_RELEASE = 'v0.3.0' +CURRENT_RELEASE = 'v{}'.format(__version__) CONF_DIR = '/etc/p2es' @@ -212,7 +213,11 @@ def log(lev, msg, exc_info=False): def main(): parser = argparse.ArgumentParser( - description="pmacct-to-elasticsearch" + description="pmacct-to-elasticsearch {}: a tool to read " + "pmacct's output and to store it into " + "ElasticSearch.".format(CURRENT_RELEASE), + epilog="Copyright (c) {} - Pier Carlo Chiodi - " + "https://pierky.com".format(2017) ) parser.add_argument( "pluginname", diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..a52d5e8 --- /dev/null +++ b/setup.py @@ -0,0 +1,83 @@ +import os +from os.path import abspath, dirname, join +from setuptools import setup, find_packages + +""" +New release procedure + +- edit pierky/p2es/version.py + +- edit CHANGES.rst + +- new files to be added to MANIFEST.in? + +- python setup.py sdist + +- twine upload dist/* + +- git push + +- edit new release on GitHub +""" + +__version__ = None + +# Allow setup.py to be run from any path +os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) + +# Get proper long description for package +current_dir = dirname(abspath(__file__)) +description = open(join(current_dir, "README.rst")).read() +changes = open(join(current_dir, "CHANGES.rst")).read() +long_description = '\n\n'.join([description, changes]) +exec(open(join(current_dir, "pierky/p2es/version.py")).read()) + +# Get the long description from README.md +setup( + name="pmacct-to-elasticsearch", + version=__version__, + + packages=["pierky", "pierky.p2es"], + namespace_packages=["pierky"], + include_package_data=True, + + license="MIT", + description="A Python script designed to read output from pmacct daemons, to process it and to store it into ElasticSearch.", + long_description=long_description, + url="https://github.com/pierky/pmacct-to-elasticsearch", + download_url="https://github.com/pierky/pmacct-to-elasticsearch", + + author="Pier Carlo Chiodi", + author_email="pierky@pierky.com", + maintainer="Pier Carlo Chiodi", + maintainer_email="pierky@pierky.com", + + install_requires=[ + ], + + scripts=["scripts/pmacct-to-elasticsearch"], + + keywords=['pmacct', 'NetFlow', 'accounting', 'BGP'], + + classifiers=[ + "Development Status :: 4 - Beta", + + "Environment :: Console", + + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "Intended Audience :: System Administrators", + "Intended Audience :: Telecommunications Industry", + + + "Operating System :: POSIX", + "Operating System :: Unix", + + "Programming Language :: Python :: 2.7", + + "Topic :: Internet :: WWW/HTTP", + "Topic :: System :: Monitoring", + "Topic :: System :: Networking", + "Topic :: System :: Networking :: Monitoring", + ], +)