#!/usr/bin/env python

# Copyright 2017 Yoav Seginer, Theo Vosse, Gil Harari, and Uri Kolodny.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# this script is used to create an application file from a cdl script,
# resolving references to other files to their actual paths.
#
# it is used in two different stages:
#
#  1. creating a compilation program, a program that would compile the
#     cdl code into a feg program
#     the generated program includes the cdl-source along with all the
#     class-files it may refer to, as well as the compilation code
#
#  2. creating the runtime program, that implements the application
#     the generated program includes the compiled feg, as well as the cdl
#     runtime environment interpreting it
#
# Both of these could use this script in either of its two modes:
#
#  1. 'js' mode - generates a single javascript file, into which all the
#         javascript files are concatenated
#
#  2. 'html' mode - generate am html file referencing the javascript file
#         with html '<script>' tags
#
#  3. 'incl' mode - generates a list of included files
#
#  4. 'make' mode - generates a make file dependency
#
# Notes:
#  Compilation stage normally generates a 'js' file, which is then run in
#   'node.js'.
#  However, when debugging the compiler it is convenient to generate an 'html'
#   version of the compilation program, and execute it in a browser, with its
#   superior debugging environment
#
# 'html' mode is used in the development environment for runtime programs,
#   so that the browser's debugger provides easy access to the run-time
#   javascript files. Changes to the run-time javascript files take affect
#   with but a reload of the browser, while changes to typescript files require
#   compiling typescript to javascript (but not cdl recompilation or
#   re-generation of the .html file by this script).
#   For batch automatic test execution, node.js is used, so 'js' mode is
#    required.
#   Also, the 'production' format, as uploaded to the build-web-site, uses
#    an 'uglified' single-file javascript referenced by an html file. This
#    single-file javascript is generated by this script using 'js' mode.
#
#
#
# Basic Functionality:
#
# This script takes two main input files: a template file and a source file.
# The template file guides the generation of the output file. For 'simple'
#  template lines, the script merely copies the line from the template file to
#  the output file.
#  However, the template files typically also uses directives and macros.
#  Directives affect an inclusion of another file. In 'html' mode, this is
#   achieved by placing a <script> tag in the output file, while in 'js' mode
#   the actual content of the included file is written to the output file.
#   In either modes, included files are recursively processed, so that nested
#   inclusion directives are handled.
#  Macros are replaced by a string which this script computes. For example,
#   the title macro, '%%title:()%%', is replaced with the value of the '--title'
#   argument to this script.
#
# confLib handling:
#
# When generating a compilation program, this script can be given the
#  path of the a --libconf' file. A libconf file describes a hierarchy of
#  conf-libs, where each conf-lib is described in a single line by its name
#  and path, separated by a colon, e.g. 'Core:../../core'
#
# In a libconf file is specified, the behavior of this script is modified in
#  the following ways:
#
# - the script translates an instance of 'var classes = ' in a file 'fn.js'
#   of confLib 'CL' to 'var CL__fn__classes ='
#   if the classes are already defined with a variable which is the basename
#   of the file, that case is recognized too:
#     'var fn =' in fn.js is converted to 'var CL__fn ='
#
# - the script emits a list with the class-lists and their confLib,
#  { confLib: "CL1", classes: [CL1__fn1_1__classes, CL1__fn1_2__classes, ...] },
#  { confLib: "CL2", classes: CL2__fn2_1__classes, ...] }, ...
#
# - the scripts recognizes a macro '%%conflibPreamble:()%%', and replaces it
#     with the inclusion of a file with a fixed name for each conf-lib in the
#     libconf file. Given a conflib path <clpath1>, the script includes the
#     file "<clpath1>/includeList.js".
#
#
# command line arguments:
#
# -o/--out_file - the output file, into which the generated .js/.html file is
#      written
#
# -t/--template - the 'template' input file, the file controlling the output
#      file format
#
# -m/--mode - js/html
#
# --includedir - add the argument to the list of directories in which include
#      files (encountered in %%include%%: macros) are searched for
#
# --langdir - set the argument as the root directory for scripts. it is added
#      to all search paths (include/classfile/image/constantfile/textfile/url)
#
# --cdldir -- like langdir, but for cdl
#
# -c/--libConf - the argument is a libConf file, detailing a hierarchy of
#      confLibs and their paths
#
# --buildInfoFile - the argument is the buildInfo file, which would become the
#      value of the %%buildinfo:()%% macro. This file typically defines the
#      version control revision, time stamp, etc
#
# --commonImageDir - when specified, images encountered in %%image:()%% macros
#      are not resolved as usual, to the first directory in the image path
#      containing the named image file. Rather, all image macros are resolved
#      to the specified commonImageDir. This is used when generating the
#      production environment, where all images are collated to a single dir
#
# --commonDataDir - like commonImageDir, but for %data:()%%
#
# --title - the argument would become the value of the %%title:()%% macro
#
# --max-include-level - do not process include lines above this level
#
# --referencedir - sets the reference directory for file name normalization
#
# --sourcedir - sets the directory that should be used for relative includes
#               from the source file; by default the directory of the source
#
# --splash-screen-url - the path to the HTML file which provides the splash
#                       screen. If not specified, the default is used.

from __future__ import print_function

import os
import sys
import re
import argparse
import pickle
import gzip
import shutil

# global variables

# js/html
mode = None

# --title --> %%title:()%%
title = None

# --splash-screen-url
splash_screen_url = None

# don't include files higher than given level
max_include_level = 999999999

# --commonImageDir
common_image_dir = None

# --commonDataDir
common_data_dir = None

# all normalized paths are relative to this directory
# (absolute paths generated in a cygwin environment are meaningless to the
# non-cygwin browser reading these paths, so paths are created as relative)
reference_dir = None

# When not none, this is supposed to be the base directory relative to the
# source file; useful for ignoring the intermediate/ prefix when processing
# .run.js files.
source_dir = None

# a list of '{ name: cl-name, path: cl-path }' generated by parsing the
#  confLib file
#
conf_lib_list = []

# for each file read/being-read, processed-files has an attribute which is the
#  file's name, and whose value is the line# we've reached within that file
processed_files = {}

# the name of the file from which the last input line was read
#  has different value for each dtype (classfile/include/constantfile etc)
last_file_name = {}

# writing into this handle writes into the output file
out_file_handle = None

# handle to the file with the dependencies
dep_file_handle = None

# search path per dtype - a list of directories
path_per_dtype = {
    'include': [],
    'classfile': [],
    'constantfile': [],
    'template': [],
    'image': [],
    'data': [],
    'foreign': [],
    'text': [],
    'url': []
}

# the 'source' file, the single positional run-time argument --> 'source'
cdl_source = None

# --buildInfoFile --> %%buildinfo:()%%
build_info_file = None

# the stack of files currently being read, which file included which file etc
#  to get us to read the current file (at the top of stack)
fn_stack = []

# inclusion_cycle_permitted, default to false
# section, defaults to 'infix' (can be  set to 'prefix'/'suffix')
# 'conf_lib' - include paths used while procesing that conf-lib
# 'sticky_conf_lib' - include paths added and remain there
dtype_property = {
    'include': {
        'conf_lib': ['design', 'func', 'automated' ],
        'html': 'script',
        'inclusion_cycle_permitted': True
    },
    'classfile': {
        'conf_lib': ['design', 'func', 'automated' ],
        'html': 'script'
    },
    'constantfile': {
        'section': 'prefix',
        'conf_lib': ['design', 'func', 'automated' ],
        'html': 'script'
    },
    'template': {},
    'image': {
        'sticky_conf_lib': ['design/img']
    }
}

# generally, text is not written directly into the output; rather it is
#  'written' by appending it to the appropriate section.
# in practice, all dtypes write to 'infix' except for 'constantfile' which
# writes into 'prefix', so that constants are defined by the time the classes
# attempt to use them
section_text = {
    'prefix': [],
    'infix': [],
    'postfix': [],
}

# conf_libs are inserted into it at 0, so that the conf-lib that ends up being
#  last (== the conf-lib of least priority) is the first to be inserted
# each entry has the following attributes:
#  'name' - conf-lib-name
#  'class_list' - the list of class_lists associated with this conf_lib
#                (e.g. ['Core__draggableClasses', 'Core__snappabaleClasses',..])
#  'constant' - the list of constant defs associated with this conf_lib
#               (as {
#                      'name': 'positioningConstants',
#                      'element': 'Core__positioningConstants'
#                   } )
#
conf_lib_by_priority = []

current_conf_lib = None

# Counter for number of screenArea declarations and var test lines
nr_screen_area = 0
nr_test = 0

# Target for make file include
make_target = None


def error_exit(msg):
    print(sys.argv[0], ": ", msg, file=sys.stderr)
    # print(str(fn_stack), file=sys.stderr)
    sys.exit(1)


# Dictionary to check that no two file names get mapped onto the same class
# variable
class_stem_names = {}

def stemname(path, conf_lib_name):
    """return the extension-less basename of path and make sure it's a legal JS identifier"""
    basename = os.path.splitext(os.path.basename(path))[0]
    class_identifier = re.sub("[^a-zA-Z0-9_]+", "_", basename)
    if conf_lib_name is None:
        return class_identifier
    conflib_class_identifier = conf_lib_name + "__" + class_identifier
    if conflib_class_identifier in class_stem_names and class_stem_names[conflib_class_identifier] != path:
        error_exit("files {} and {} map to the same variable name".format(path, class_stem_names[conflib_class_identifier]))
    class_stem_names[class_identifier] = path
    return class_identifier


def get_arg_parser():
    parser = argparse.ArgumentParser(description='cdl script aggregator')
    parser.add_argument('-o', '--out_file', help='output file name',
                        required=True)
    parser.add_argument('-d', '--dep_file', help='dependency file name')
    parser.add_argument('--dep_target', help='dependency target')
    parser.add_argument('cdl_source', help='input cdl script file')
    parser.add_argument('-t', '--template', help='template file name')

    parser.add_argument('-m', '--mode', choices=['js', 'html', 'incl', 'make'],
                        help='processing mode, "js"/"html"/"incl"/"make"', required=True)

    parser.add_argument('--includedir', help='include directory',
                        action='append')

    parser.add_argument('--langdir', help='root of the lang directory')

    parser.add_argument('--cdldir', help='root of the cdl apps and classes directory')

    parser.add_argument('--referencedir', help='set reference directory; by default cwd')

    parser.add_argument('--sourcedir', help='set directory for includes from source file')

    parser.add_argument('-c', '--libConf', help='library configuration')

    parser.add_argument('--buildInfoFile',
                        help='path of the build-info file, with rev#/date/etc')

    parser.add_argument('--resourceOutFile', help='path for writing used resources')

    parser.add_argument('--resourceUseFile', help='path for used resources')

    parser.add_argument('--max-include-level', help='suppress indirect includes above level')

    parser.add_argument('--commonImageDir', help='replaces all references in image macros')

    parser.add_argument('--commonDataDir', help='replaces all references in data macros')

    parser.add_argument('--title')

    parser.add_argument('--splash-screen-url', help='a URL pointing at the HTML page which should server as a splash screen')
    
    return parser


def annotate(msg):
    print(msg)


def set_reference_dir(rd):
    global reference_dir
    if rd is None:
        reference_dir = os.path.realpath(os.getcwd())
    else:
        reference_dir = os.path.realpath(rd)


def set_source_dir(sd):
    global source_dir
    if sd is not None:
        source_dir = os.path.realpath(sd)


def normalize_path(path):
    real_path = os.path.realpath(path)
    return os.path.relpath(real_path, reference_dir)


def push_include_file(dtype, fn):
    global fn_stack
    global dtype_property

    if dtype not in dtype_property:
        error_exit("unknown type '" + dtype + "'")

    cycle_permitted = dtype_property[dtype]['inclusion_cycle_permitted'] if 'inclusion_cycle_permitted' in dtype_property[dtype] else False
    if not cycle_permitted:
        if fn in fn_stack:
            error_exit("inclusion cycle for '" + fn + "'")

    fn_stack.append(fn)

def pop_include_file(dtype, fn):
    global fn_stack

    popped_file = fn_stack.pop()
    if popped_file != fn:
        error_exit("internal error: wrong file in pop (" +
                   fn + "!=" + popped_file + ")")

    if len(fn_stack) == 1:
        str = get_output_str()
        write_str(str)

def get_last_file_name(dtype):
    global last_file_name
    if dtype in last_file_name:
        return last_file_name[dtype]
    return ""


def set_last_file_name(dtype, fn):
    global last_file_name
    last_file_name[dtype] = fn


def append_root_path(path):
    global path_per_dtype
    if (path != None):
        paths = path.split(";")
        path_per_dtype['include'].extend(paths)
        path_per_dtype['classfile'].extend(paths)
        path_per_dtype['constantfile'].extend(paths)
        path_per_dtype['image'].extend(paths)
        path_per_dtype['data'].extend(paths)
        path_per_dtype['foreign'].extend(
            map(lambda path: os.path.join(path, "external", "foreignInterface"),
            paths))
        path_per_dtype['text'].extend(paths)
        path_per_dtype['url'].extend(paths)


def set_include_path(path_list):
    global path_per_dtype
    if (path_list != None):
        path_per_dtype['include'].extend(path_list)


def set_build_info_file(path):
    global build_info_file
    if path != None:
        build_info_file = path


# Creates a directory at path if it doesn't exist, or is not a directory
def mkdir_if_not_exists(path):
    if not os.path.isdir(path):
        if os.path.exists(path):
            os.remove(path)
        os.makedirs(path)


def set_common_image_dir(path):
    global common_image_dir
    if path != None:
        common_image_dir = path
        mkdir_if_not_exists(path)


def set_common_data_dir(path):
    global common_data_dir
    if path != None:
        common_data_dir = path
        mkdir_if_not_exists(path)


def set_cdl_source(fn):
    global cdl_source
    cdl_source = fn


def get_cdl_source():
    global cdl_source
    return cdl_source
    

def set_mode(m):
    global mode
    mode = m


def get_mode():
    global mode
    return mode


def set_out_file(fn):
    global out_file_handle
    out_file_handle = open(fn, 'w');


def set_dep_file(fn):
    global dep_file_handle
    if fn != None:
        dep_file_handle = open(fn, 'w');


# Name of the file that stores the resources used in the current input
res_out_file_name = None
# used resources per resource type
used_resources = { "foreign": set(), "font": set(), "text": set() }
# flag to indicate use of external resources
does_use_resources = False


def set_res_out_file(fn):
    global res_out_file_name
    res_out_file_name = fn


def add_resource_usage(res_type, res_uri):
    global used_resources, does_use_resources
    if res_uri in used_resources.get(res_type):
        return
    used_resources.get(res_type).add(res_uri)
    does_use_resources = True


def write_resource_usage():
    global used_resources, res_out_file_name, does_use_resources
    if res_out_file_name != None:
        if does_use_resources:
            if os.path.exists(res_out_file_name):
                with open(res_out_file_name, 'rb') as input:
                    dict = pickle.load(input)
                    if dict == used_resources:
                        return
            with open(res_out_file_name, 'wb') as output:
                pickle.dump(used_resources, output, pickle.HIGHEST_PROTOCOL)
        elif os.path.exists(res_out_file_name):
            os.remove(res_out_file_name)

def set_res_use_file(fn):
    global used_resources
    if fn != None and os.path.exists(fn):
        with open(fn, 'rb') as input:
            used_resources = pickle.load(input)

def write_font_urls(directive_prefix):
    global used_resources
    mode = get_mode()
    if mode == 'html':
        for url in used_resources.get('font'):
            write_str(directive_prefix + '<link rel="stylesheet" type="text/css" href="' + url + '">\n')

def set_title(str):
    global title
    if str == None:
        title = ""
    else:
        title = str

def set_splash_screen_url(str):
    global splash_screen_url
    if str == None:
        splash_screen_url = ""
    else:
        splash_screen_url = str
        
def set_make_target(arg, deflt):
    global make_target
    if arg == None:
        make_target = deflt
    else:
        make_target = arg

def set_max_include_level(arg):
    global max_include_level
    if arg is not None:
        max_include_level = arg

def write_str(str):
    global out_file_handle
    out_file_handle.write(str)

def write_dep(fn):
    global make_target
    global dep_file_handle
    if dep_file_handle != None:
        dep_file_handle.write(make_target + ': ' + fn + '\n')

#
# called just before concatenating the sections (prefix/infix/postfix) into the
#  output file
# generates code to merge constants defined in several conf-lib constantfiles
#  into a single constant (assumes js mode)
def output_section_hook():
    constant_dict = {}
    for clentry in conf_lib_by_priority:
        cl_const_list = clentry['constant']
        for const_entry in cl_const_list:
            const_name = const_entry['name']
            const_elem = const_entry['element']
            if const_name not in constant_dict:
                constant_dict[const_name] = []
            constant_dict[const_name].append(const_elem)
        clentry['constant'] = []

    for const_name in constant_dict:
        const_merge_def = 'var ' + const_name + ' = ' + \
                          'mergeCdlConstants(\n\t[\n\t\t'
        const_merge_def += ',\n\t\t'.join(constant_dict[const_name])
        const_merge_def += '\n\t]\n)\n'

        section_print('constantfile', const_merge_def)


def get_output_str():
    global section_text

    output_section_hook()

    str = "".join(section_text['prefix'])
    str += "".join(section_text['infix'])
    str += "".join(section_text['postfix'])

    section_text['prefix'] = []
    section_text['infix'] = []
    section_text['postfix'] = []

    return str

def section_print(dtype, line):
    global section_text

    if dtype == 'template':
        write_str(line)
    else:
        if 'section' in dtype_property[dtype]:
            section = dtype_property[dtype]['section']
        else:
            section = 'infix'

        section_text[section].append(line)

# annotate js output file with the input fn/line#
def gen_filename_and_line_number(dtype, fn):
    global processed_files

    mode = get_mode()
    if mode == 'html':
        return

    if get_last_file_name(dtype) != fn:
        if mode == 'js':
            section_print(dtype, "//# " + fn + ":" + str(processed_files[fn]) +
                          '\n');
        set_last_file_name(dtype, fn)

    processed_files[fn] = processed_files[fn] + 1

def find_file_in_path(dtype, basename):
    global path_per_dtype

    if basename.startswith("."):
        if (os.path.isfile(basename)):
            return basename
    else:
        path = path_per_dtype[dtype]
        for dirp in path:
            file_path = os.path.join(dirp, basename)
            if (os.path.isfile(file_path)):
                return file_path

    if dtype == "image":
        print("could not find path for image " + basename, file=sys.stderr)
        return basename

    path_list = ":".join(path)
    error_exit("could not find path for <" + dtype + "> file '" +
               basename + "' in path '" + path_list + "'");

# add the conf-lib path for dtypes where this is required (e.g. classfile,
#  include)
def push_conf_lib_path(path):
    global path_per_dtype
    global dtype_property

    for dtype in dtype_property:
        if 'conf_lib' in dtype_property[dtype]:
            for sub_path in dtype_property[dtype]['conf_lib']:
                dpath = os.path.join(path, sub_path)
                path_per_dtype[dtype].append(dpath)


def pop_conf_lib_path(path):
    global path_per_dtype
    global dtype_property

    for dtype in dtype_property:
        if 'conf_lib' in dtype_property[dtype]:
            for sub_path in reversed(dtype_property[dtype]['conf_lib']):
                dpath = os.path.join(path, sub_path)
                apath = path_per_dtype[dtype].pop()

                if apath != dpath:
                    error_exit("popped path does not match pushed path")

# add search path per the currently processed conf-lib, and leave it there so
#  that following files may use it too (also outside the current conf-lib)
#  for the appropriate dtypes (e.g. image, data)
def add_conf_lib_sticky_path(path):
    global dtype_property

    for dtype in dtype_property:
        if 'sticky_conf_lib' in dtype_property[dtype]:
            for sub_path in dtype_property[dtype]['sticky_conf_lib']:
                dpath = os.path.join(path, sub_path)
                path_per_dtype[dtype].append(dpath)


def add_conf_lib(priority, name, path):
    global conf_lib_by_priority
    global current_conf_lib

    conf_lib_by_priority.insert(0,
                                {
                                    'name': name,
                                    'class_list': [],
                                    'constant': []
                                })

    current_conf_lib = { 'priority': priority, 'name': name, 'path': path }

    if name == None:
        return

    push_conf_lib_path(path)
    conf_lib_include = os.path.join(path, "includeList.js")
    if os.path.exists(conf_lib_include):
        process_file('include', conf_lib_include, os.path.dirname(conf_lib_include))
    pop_conf_lib_path(path)

    current_conf_lib = None

            
def set_lib_conf(lib_conf):
    global conf_lib_list

    if lib_conf == None or not os.path.exists(lib_conf):
        return

    with open(lib_conf) as lib_conf_handle:
        for line in lib_conf_handle:

            # remove comments
            match = re.search('^[^#]*', line)
            line = match.group()

            # skip empty lines
            if re.search('^\s*$', line):
                continue

            # parse (allowing spaces)
            # <confLibName>:<confLibPath>
            match = re.search('^\s*(?P<name>[a-zA-Z0-9_]+)\s*:' +
                                        '\s*(?P<path>[^\s]*)\s*$', line)
            if (not match.group('name')) or (not match.group('path')):
                error_exit("libConf file syntax error: '" + line + "'");

            lcname = match.group('name')
            lcpath = match.group('path')

            conf_lib_list.append({ 'name': lcname, 'path': lcpath })


def gen_conf_lib_preamble():
    for conf_lib in conf_lib_list:
        add_conf_lib_sticky_path(conf_lib['path'])

    for idx, conf_lib in reversed(list(enumerate(conf_lib_list))):
        clpriority = len(conf_lib_list) - idx
        add_conf_lib(clpriority, conf_lib['name'], conf_lib['path'])

    add_conf_lib(0, None, None)

    preamble = get_output_str()

    return preamble


def process_directive(line, directive_fn, linenr, basedir):
    global make_target, used_resources

    filename = None
    filenames = None
    relative_dir = None

    match = re.search('^([^a-z]*)%%([a-z]+)%%:\s*([^\s]*)\s*$', line)
    if match == None or len(match.groups()) != 3:
        error_exit(directive_fn + ':' + str(linenr) + ': directive has invalid syntax: ' + line)
    directive_prefix = match.group(1)
    directive = match.group(2)
    basename = match.group(3)

    stdmatch = re.search('^<(.*)>$', basename)
    quotematch = re.search('^"(.*)"$', basename)
    # tildematch = re.search('^~/(.*)$', basename)

    if stdmatch != None:
        filename = find_file_in_path(directive, stdmatch.group(1))
    elif quotematch != None:
        filename = os.path.join(basedir, quotematch.group(1))
        if not os.path.isfile(filename):
            filename = find_file_in_path(directive, quotematch.group(1))
    elif basename == 'source':
        filename = get_cdl_source()
        if source_dir is not None:
            relative_dir = source_dir
    elif basename == 'foreign':
        filenames = []
        for fn in used_resources.get("foreign"):
            filenames.append(find_file_in_path('foreign', fn))
    elif basename == 'fonturls':
        write_font_urls(directive_prefix)
        return
    # elif tildematch != None:
    #     filename = os.path.join(get_root_dir(), tildematch.group(1))
    else:
        print('basename="' + basename + '"')

    if get_mode() == 'incl':
        print(directive, filename)

    if filename is not None:
        if relative_dir is None:
            relative_dir = os.path.dirname(filename)
        process_file(directive, filename, relative_dir)
    elif filenames is not None:
        for filename in filenames:
            process_file(directive, filename, os.path.dirname(filename))
    else:
        error_exit('invalid directive: ' + line)


# Only compress svg images
def use_compression_for_image(filename):
    return filename.endswith(".svg")


# Compress all data files
def use_compression_for_data(filename):
    return True


# Stores which resource has been copied to which path; avoids duplicate copies
# and resolves faster
copied_resources = {}
# Stores which path is the target for which resource; avoids duplicate naming
resource_targets = {}

def add_copied_resource(resource_hash, path):
    global copied_resources
    if path in resource_targets and resource_targets[path] != resource_hash:
        error_exit("{} is the target for both {} and {}".format(
            path, resource_targets[path], resource_hash
        ))
    copied_resources[resource_hash] = path
    resource_targets[path] = resource_hash


# Returns the path to the file from the macro. When common_dir has been set,
# copies the file to that directory, compressing it when the extension allows
# it, but only when the source file is newer.
def copy_and_compress(type, macro_arg, use_compression_fun, common_dir):
    global copied_resources
    resource_hash = type + ':' + macro_arg
    if resource_hash in copied_resources:
        return copied_resources[resource_hash]
    src_path = find_file_in_path(type, macro_arg)
    if common_dir == None:
        add_copied_resource(resource_hash, src_path)
        return src_path
    out_path = os.path.join(common_dir, os.path.basename(macro_arg))
    if not os.path.exists(src_path):
        print("{0} does not exist: {1}".format(type, src_path), file=sys.stderr)
        add_copied_resource(resource_hash, out_path)
        return out_path
    use_compression = use_compression_fun(macro_arg)
    if out_path == src_path:
        add_copied_resource(resource_hash, src_path)
        return out_path # In case someone puts the images in the common_dir
    target_path = out_path
    if use_compression:
        target_path += '.gz'
    if not os.path.exists(target_path) or os.path.getmtime(target_path) < os.path.getmtime(src_path):
        if use_compression:
            with open(src_path, 'rb') as f_in, gzip.open(target_path, 'wb') as f_out:
                shutil.copyfileobj(f_in, f_out)
        else:
            with open(src_path, 'rb') as f_in, open(target_path, 'wb') as f_out:
                shutil.copyfileobj(f_in, f_out)
    add_copied_resource(resource_hash, src_path)
    return out_path


# format: %%image:(url)%%. Behaves like process_image_macro.
# Calls copy_and_compress for an image
def process_image_macro(macro_name, macro_args):
    global common_image_dir

    return copy_and_compress('image', macro_args[0], use_compression_for_image, common_image_dir)


# format: %%font:(fontFamily,url)%%, no comma in the font name, no superfluous spaces
def process_font_macro(macro_name, macro_args):
    if len(macro_args) < 2:
        error_exit('font macro should have two arguments')
    url = ",".join(macro_args[1:]) # in case the URL constains commas
    add_resource_usage('font', url)
    return macro_args[0]


# format: %%data:(url)%%. Behaves like process_image_macro.
# Calls copy_and_compress for a data file
def process_data_macro(macro_name, macro_args):
    global common_data_dir

    return copy_and_compress('data', macro_args[0], use_compression_for_data, common_data_dir)


def process_buildinfo_macro(macro_name, macro_args):
    global build_info_file

    return build_info_file


def process_conf_lib_preamble_macro(macro_name, macro_args):
    push_include_file('template', '--conf-lib-include--')
    str = '\n' + gen_conf_lib_preamble()
    pop_include_file('template', '--conf-lib-include--')
    return str


def process_title_macro(macro_name, macro_args):
    global title
    return title

def process_splash_screen_url_macro(macro_name, macro_args):
    global splash_screen_url
    return normalize_path(find_file_in_path('url', splash_screen_url))

def process_classes_macro(macro_name, macro_args):
    global conf_lib_by_priority

    return "\n" + \
        ",\n".join(
            map(
                lambda x: "\t{\n\t\tname: '" +
                ("" if x['name'] == None else x['name']) +
                "',\n\t\tclasses: [\n\t\t\t" +
                ",\n\t\t\t".join(x['class_list']) +
                "\n\t\t]\n\t}",
                conf_lib_by_priority
            )
        ) + "\n"


def process_textfile_macro(macro_name, macro_args):
    if len(macro_args) != 1:
        error_exit('textfile macro should have one argument')
    src_path = find_file_in_path('text', macro_args[0])
    if get_mode() == 'incl':
        print('textfile', src_path)
        return ""
    str = ""
    with open(src_path) as input_handle:
        for line in input_handle:
            str += "\\n" + line[:-1].replace('\\', '\\\\').replace('"', '\\"')
    return str[2:]


def process_url_macro(macro_name, macro_args):
    if len(macro_args) != 1:
        error_exit('textfile macro should have one argument')
    return find_file_in_path('url', macro_args[0])


def process_macro(dtype, line, fn, linenr, match):
    macro_name = match.group(1)
    macro_arg_str = match.group(2)

    # extract arguments
    macro_args = re.findall('[^,]+', macro_arg_str)

    if macro_name == 'image':
        macro_subst = process_image_macro(macro_name, macro_args)
    elif macro_name == 'data':
        macro_subst = process_data_macro(macro_name, macro_args)
    elif macro_name == 'font':
        macro_subst = process_font_macro(macro_name, macro_args)
    elif macro_name == 'buildinfo':
        macro_subst = process_buildinfo_macro(macro_name, macro_args)
    elif macro_name == 'conflibPreamble':
        macro_subst = process_conf_lib_preamble_macro(macro_name, macro_args)
    elif macro_name == 'title':
        macro_subst = process_title_macro(macro_name, macro_args)
    elif macro_name == 'splashScreenUrl':
        macro_subst = process_splash_screen_url_macro(macro_name, macro_args)
    elif macro_name == 'classes':
        macro_subst = process_classes_macro(macro_name, macro_args)
    elif macro_name == 'textfile':
        macro_subst = process_textfile_macro(macro_name, macro_args)
    elif macro_name == 'url':
        macro_subst = process_url_macro(macro_name, macro_args)
    else:
        error_exit(fn + ':' + str(linenr) + ": don't know (yet) how to handle macro '" + macro_name +
                   "' in '" + line + "'")

    if macro_subst == None:
        error_exit(fn + ':' + str(linenr) + ': empty subst')
    return macro_subst


def get_current_conf_lib_name():
    global current_conf_lib
    if current_conf_lib == None or current_conf_lib['name'] == None:
        conf_lib_name = ""
    else:
        conf_lib_name = current_conf_lib['name']
    return conf_lib_name


def verify_current_conf_lib(conf_lib_name):
    cblp_name = conf_lib_by_priority[0]['name']
    if cblp_name == None:
        cblp_name = ""

    if cblp_name != conf_lib_name:
        error_exit('confLib names do not match')


def process_class_def(dtype, line, fn):
    """replace 'var classes =' with 'var <CL>__<fn>__classes ='
    where <CL> is the current confLib (may be empty) and <fn> is the current
    source file name"""

    global conf_lib_by_priority

    conf_lib_name = get_current_conf_lib_name()
    verify_current_conf_lib(conf_lib_name)

    mclass_name = conf_lib_name + '__' + stemname(fn, conf_lib_name) + '__classes'
    mclass_def = 'var ' +  mclass_name + ' ='
    match = re.search('^\s*var[^=]*=(.*)$', line)
    mclass_def = mclass_def + match.group(1) + "\n"

    section_print(dtype, mclass_def)

    conf_lib_by_priority[0]['class_list'].append(mclass_name)


def process_constant_def(dtype, line, fn):
    """
replace
   'var xxxConstants = { ... };'
 with
   'var <confLib1>__xxxConstants = { ... };'
 and then, at the end of the 'constantfile' section append
   'var xxxConstants = mergeCdlConstants(
       <confLib1>__xxxConstants,
       <confLib2>__xxxConstants,
       ...
    );'
 (ordered by confLib priority) to allow higher priority confLibs to
 overwrite constants defined in lower priority confLibs, such that the
 affect reaches back into the lower priority confLib. For example, if Core has

CellConstants = {
  width: 5
}

Cell: { position: { width: CellConstants.width } }

and Mon1 has
CellConstants = {
  width: 2
}

then setting CellConstants.width to 2 must occur before including Core::Cell

a constant definition is also identified as
var xxx = { // %%constantdef%%

"""
    conf_lib_name = get_current_conf_lib_name()
    verify_current_conf_lib(conf_lib_name)

    # neutralize processed %%constantdef%% by converting %% to %-
    constdef_match = re.search('^(.*//.*)%%constantdef%%(.*)$', line)
    if constdef_match:
        line = constdef_match.group(1) + '%-constantdef-%' + \
               constdef_match.group(2)
    match = re.search('^\s*var\s+([a-zA-Z0-9_]+)\s*=(.*)$', line)
    if (not match) or (not match.group(1)) or (not match.group(2)):
        error_exit('constant_def: parse failure (' + line + ')')

    const_name = match.group(1)
    mconst_name = conf_lib_name + '__' + const_name

    mconst_def = 'var ' + mconst_name + ' =' + match.group(2) + "\n"

    section_print(dtype, mconst_def)

    conf_lib_by_priority[0]['constant'].append({
        'name': const_name,
        'element': mconst_name
    })

# The pattern for macros
macro_re = re.compile('%%([a-zA-Z0-9_]*):\(([^%()]*)\)%%')
# The pattern for includes
include_re = re.compile('^[^a-z]*%%[a-z]+%%:')

# Returns a string indicating the line type
# - 'class' when the line is var classes/stemname = ...
# - 'screen' when the line is var screenArea = ...
# - '' otherwise
def process_line(dtype, line, fn, linenr, basedir):
    line = line.rstrip('\n')
    line += '\n'
    mode = get_mode()

    line = macro_re.sub(lambda match_group: process_macro(dtype, line, fn, linenr, match_group), line)

    if include_re.search(line):
        process_directive(line, fn, linenr, basedir)

    elif dtype == 'classfile' and (re.search('^\s*var\s+classes\s*=', line) or \
                       re.search('^\s*var\s*' + stemname(fn, None) + '\s*=', line)):
        if mode == 'js':
            process_class_def(dtype, line, fn)
        return 'class'

    elif (dtype == 'constantfile' and \
             re.search('^\s*var\s+[a-zA-Z0-9_]+[cC]onstants\s*=', line)) \
          or \
            re.search('\s*var\s+[a-zA-Z0-9_]+\s*=.*//.*%%constantdef%%', line):
        if mode == 'js':
            process_constant_def(dtype, line, fn)
        return 'constant'

    else:
        if dtype == 'template' or get_mode() == 'js':
            section_print(dtype, line)
        if re.search('^\s*var\s+screenArea\s*=', line):
            return 'screen'
        if re.search('^\s*var\s+test\s*=', line):
            return 'test'
    return ''

def process_file(dtype, filename, basedir):
    global processed_files
    global nr_screen_area
    global nr_test
    global max_include_level

    class_found = False
    screen_area_found = False
    test_found = False
    constant_found = False
    mode = get_mode()
    linenr = 1

    if dtype == 'foreign':
        add_resource_usage('foreign', filename)
        return

    normalized_filename = normalize_path(filename)

    if dtype == 'include' and len(fn_stack) >= max_include_level:
        return

    write_dep(normalized_filename)

    # annotate("process_file: type='" + dtype + "' filename='" + filename +
    #    "' (" + normalized_filename + ")")

    push_include_file(dtype, normalized_filename)

    if normalized_filename not in processed_files:
        processed_files[normalized_filename] = 1

        try:
            with open(normalized_filename) as input_handle:
                for line in input_handle:
                    gen_filename_and_line_number(dtype, normalized_filename)
                    line_type = process_line(dtype, line, normalized_filename, linenr, basedir)
                    if line_type == 'class':
                        if class_found:
                            error_exit("two class definitions in " + normalized_filename)
                        class_found = True
                    elif line_type == 'screen':
                        if screen_area_found:
                            error_exit("two screenAreas in " + normalized_filename)
                        screen_area_found = True
                        nr_screen_area += 1
                    elif line_type == 'test':
                        if test_found:
                            error_exit("two test definitions in " + normalized_filename)
                        test_found = True
                        nr_test += 1
                    elif line_type == 'constant':
                        constant_found = True
                    # if mode == 'incl' and (class_found or screen_area_found or test_found or constant_found):
                    #     break # Stop scanning file for includes
                    linenr += 1
        except IOError:
            print("cannot open file: " + normalized_filename + " from " + fn_stack[len(fn_stack)-2], file=sys.stderr)
            sys.exit(1)

        if mode == 'html':
            if 'html' in dtype_property[dtype]:
                html_handling = dtype_property[dtype]['html']
                if html_handling == 'script':
                    section_print(dtype, '\t<script src="' +
                                  normalized_filename + '">')
                    section_print(dtype, '</script>\n')

        if dtype == 'classfile' and mode != 'incl' and \
              not screen_area_found and not class_found and not test_found:
            print("WARNING: no screenArea, classes or test defined in " + normalized_filename)

    pop_include_file(dtype, normalized_filename)


def main():
    global reference_dir

    parser = get_arg_parser()
    args = parser.parse_args()
    mode = args.mode

    set_reference_dir(args.referencedir)

    set_mode(mode)

    set_source_dir(args.sourcedir)
    append_root_path(args.langdir)
    append_root_path(args.cdldir)
    set_include_path(args.includedir)
    set_dep_file(args.dep_file)
    set_res_out_file(args.resourceOutFile)
    set_res_use_file(args.resourceUseFile)

    set_build_info_file(args.buildInfoFile)

    set_common_image_dir(args.commonImageDir)
    set_common_data_dir(args.commonDataDir)

    cdl_source = args.cdl_source

    set_cdl_source(cdl_source)

    template = args.template

    out_file = args.out_file

    set_make_target(args.dep_target, out_file)

    set_max_include_level(args.max_include_level)

    set_out_file(out_file)

    libConf = args.libConf

    set_lib_conf(libConf)

    set_title(args.title)

    set_splash_screen_url(args.splash_screen_url)
    
    process_file('template', template, os.path.dirname(template))

    write_resource_usage()

    if mode == 'js' and out_file.endswith(".comp.js.tmp") and \
       nr_screen_area != 1:
        error_exit("no screenArea definition")

    sys.exit(0)

if __name__ == "__main__":
   main()
