generate.py

#!/usr/bin/env python
# -*- coding: utf8 -*-

# Copyright © 2017  Rinat Ibragimov
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# ---
# Generates CMakeLists.txt file for Nginx.
#
# This script works by intercepting compiler by setting CC and CXX environment
# variables to point to a specifically crafted shell scripts which record
# how Nginx build system calls compiler. Then, data are processed, and commands
# are converted to the corresponding CMake statements.
#
# Usage.
#
# Place script into the root directory of nginx sources. Call it with all
# parameters you want ./configure to be called with. Say,
#
#   $ python generate.py --with-http_realip_module --with-stream=dynamic
#
# There will be two directories created, "tmp" and "out". Former contains
# temporary files you don't need. Latter, "out", contains resulting files:
# CMakeLists.txt and directory "objs" with files, generated by Nginx build
# scripts. You'll need both.
#
# Then, craft a final result by coping nginx sources directory "src" and
# generated files into directory. That's what that directory should look like:
#
#   ├── CMakeLists.txt
#   ├── obj
#   │   ├── ngx_auto_config.h
#   │   ├── ngx_auto_headers.h
#   │   ├── ngx_http_geoip_module_modules.c
#   │   ├── ngx_modules.c
#   │   ├── ngx_stream_geoip_module_modules.c
#   │   └── ngx_stream_module_modules.c
#   └── src
#       ├── core
#       │   └── ...
#       ├── event
#       │   └── ...
#       ├── http
#       │   └── ...
#       .
#       .
#       .
#
# Presense of files named "*_module_modules.c" depends whenever you are using
# dynamic modules.
#
# Generated CMakeLists.txt loads file PreLists.txt at the beginning and file
# PostLists.txt at the end. Both of them are optional. You may to want, for
# example, add
#
#   add_definitions(-DNGX_PREFIX="${CMAKE_INSTALL_PREFIX}/")
#
# to the PreLists.txt to change NGX_PREFIX which was remembered during
# configuration step. PostLists.txt is the right place for linking nginx binary
# with a your custom statically linked module, and so on.
#
# There is also BINARY_PREFIX internal variable which could be used to change
# "nginx" binary name. Setting it to "custom-" will change binary name to
# "custom-nginx".
#
# If you have static module for nginx, you need placeholders in module lists in
# ngx_modules.c. There is a helper stub module, in "custom_module_stub"
# directory. Change names there to those you use, and then add it via
# "--add-module" configuration parameter.
#
# ---

import os
import shlex
import stat
import subprocess
import sys


# Paths.
workdir = os.path.join(os.getcwd(), 'tmp')
outdir = os.path.join(os.getcwd(), 'out')
outdir_objs = os.path.join(outdir, 'objs')
workdir_record_c = os.path.join(workdir, 'record-c')
workdir_record_cplusplus = os.path.join(workdir, 'record-c++')
workdir_configure_log = os.path.join(workdir, 'configure-log')
workdir_build_log = os.path.join(workdir, 'build-log')

def mkdir_always(path):
    try:
        os.mkdir(path)
    except OSError:
        # mkdir() could throw is directory exists.
        pass

mkdir_always(workdir)
mkdir_always(outdir)
mkdir_always(outdir_objs)

# Prepare intercepter scripts.
with open(workdir_record_c, 'w') as f:
    f.write('#!/bin/sh\necho record-c $@ >> $TRACE_LOG\nexec gcc $@\n')
    os.fchmod(f.fileno(), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)

with open(workdir_record_cplusplus, 'w') as f:
    f.write('#!/bin/sh\necho record-c++ $@ >> $TRACE_LOG\nexec g++ $@\n')
    os.fchmod(f.fileno(), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)

def truncate_file(path):
    with open(path, 'w') as f:
        # Just truncate file
        pass

# Cleanup trace files. They're expected to be empty.
truncate_file(workdir_configure_log)
truncate_file(workdir_build_log)

# Prepare environment.
env = dict(os.environ)
env['CC'] = workdir_record_c
env['CXX'] = workdir_record_cplusplus

# Call configure script.
env['TRACE_LOG'] = workdir_configure_log
subprocess.Popen(['/bin/sh', './configure'] + sys.argv[1:], env=env).wait()

# Call make.
env['TRACE_LOG'] = workdir_build_log
subprocess.Popen(['make', '-j1'], env=env).wait()

# Copy generated sources.
subprocess.call(['cp objs/*.c objs/*.h "{}"'.format(outdir_objs)], shell=True)

# Process build log.
with open(workdir_build_log) as f:
    lines = f.readlines()

# All warning flags ever seen during compilation.
warning_flags = set()

# All include directories even seen during compilation.
include_directories = set()

# Maps target name to a set of libraries it should be linked with.
target_link_libraries = dict()

# All targets that are executables.
target_executables = set()

# All targets that are shared libraries (.so).
target_shared_objects = set()

# Maps object to a set of files it was created from.
target_objects = dict()

# Additional compiler definitions.
target_definitions = set()

want_fPIC = False

for line in lines:
    line = shlex.split(line)

    # First element is always "record-c".
    line.pop(0)

    destination_file = ""
    output_name = ""
    output_type = "executable"
    libraries = set()
    source_files = set()

    try:
        while True:
            item = line.pop(0)

            if item == '-c':
                output_type = "object"

            elif item == '-pipe':
                # non-essential
                pass

            elif item[:2] == '-O':
                # That's an optimization flag. Ignoring.
                pass

            elif item[:4] == '-Wl,':
                if item == '-Wl,-E':
                    libraries.add('-Wl,-E')
                else:
                    print("unknown linker flag")
                    sys.exit(2)

            elif item[:2] == '-W':
                # That's a warning flag.
                warning_flags.add(item)

            elif item[:2] == '-g':
                # That's a debug information flag. Ignoring.
                pass

            elif item[:2] == '-I':
                # That's an include flag. Directory can be either glued to -I,
                # or come in a separate parameter.
                if len(item) == 2:
                    # Separate.
                    include_dir = line.pop(0)
                    include_directories.add(include_dir)
                else:
                    # Glued. Throw away first two bytes, take the rest.
                    include_directories.add(item[2:])

            elif item == '-o':
                # Compilation output.
                output_name = line.pop(0)

            elif item[:2] == '-D':
                target_definitions.add(item[2:])

            elif item[:2] == '-l':
                # Dynamic libraries.
                libraries.add(item)

            elif item[-2:] == '.a':
                # Static libraries.
                libraries.add(item)

            elif item == '-fPIC':
                want_fPIC = True

            elif item == '-shared':
                # This line generates shared library
                output_type = "shared"

            elif item[:2] == '-L':
                # Libraries path. Usual libraries are expected at standard
                # paths, so this is something custom. Let's avoid putting those
                # into CMake directives here. Those can be added manually.
                pass

            else:
                source_files.add(item)

    except IndexError:
        # no items in the line left
        pass

    if output_type == "shared":
        target_shared_objects.add(output_name)
    elif output_type == "executable":
        target_executables.add(output_name)
    else:
        # Ordinary object file, no need to store it
        pass

    target_link_libraries[output_name] = libraries
    target_objects[output_name] = source_files


print("Generating CMakeLists.txt")

cmakelists_txt = "# Autogenerated file.\n\n"
cmakelists_txt += "cmake_minimum_required(VERSION 2.8)\n\n"
cmakelists_txt += "add_definitions(-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64)\n\n"
cmakelists_txt += "include(PreLists.txt OPTIONAL)\n\n"

if want_fPIC:
    cmakelists_txt += "add_definitions(-fPIC)\n"

# Additional compiler definitions.
for item in target_definitions:
    cmakelists_txt += 'add_definitions(' + item + ')\n'

cmakelists_txt += "\n# warning flags: " + " ".join(warning_flags) + "\n\n"

for item in include_directories:
    cmakelists_txt += "include_directories(" + item + ")\n"

def process_targets(target_list, directive, directive_suffix):
    s = ""
    for target in target_list:
        target_basename = os.path.splitext(os.path.basename(target))[0]

        if directive == 'add_executable':
            target_basename = '${BINARY_PREFIX}' + target_basename

        s += directive + "(" + target_basename + directive_suffix + "\n"

        # Collect sources.
        sources_list = []
        for obj in target_objects[target]:
            obj_sources = target_objects[obj]
            for source_file in obj_sources:
                sources_list.append("    " + source_file)

        # Append them in sorted order.
        s += "\n".join(sorted(sources_list)) + "\n"
        s += ")\n\n"

        s += "target_link_libraries(" + target_basename + "\n"
        for lib in target_link_libraries[target]:
            s += "    "  + lib + "\n"
        s += ")\n\n"

        if directive == "add_library":
            s += 'set_target_properties(' + target_basename;
            s += ' PROPERTIES PREFIX "" OUTPUT_NAME ' + target_basename
            s += ')\n\n'

        s += 'install(TARGETS ' + target_basename + ' DESTINATION '
        s += 'lib' if directive == 'add_library' else 'bin'
        s += ')\n\n'

    return s

# Process targets.
cmakelists_txt += '\n'
cmakelists_txt += process_targets(target_executables, 'add_executable', '')
cmakelists_txt += process_targets(target_shared_objects, 'add_library',
                                  ' SHARED')

cmakelists_txt += "include(PostLists.txt OPTIONAL)\n\n"

with open(os.path.join(outdir, 'CMakeLists.txt'), 'wb') as f:
    f.write(cmakelists_txt.encode('UTF-8'))

print("done")