pavement.py

# -*- coding: utf-8 -*-

"""
Example Usage
=============

The following commands can be run from the root directory of the Mercurial
repo. To run ``paver``, however, you'll need to do ``easy_install Paver``.
Most of the following commands accept other arguments; see ``command --help``
for more information, or ``paver help`` for a list of all the valid commands.

    ``paver build``
        Builds the project. This essentially just runs a bunch of other tasks,
        like ``pylint`` and ``tinymce_zip``, etc.
    ``paver pylint``
        Runs PyLint on the project.
    ``paver tinymce_zip``
        Builds the TinyMCE zip file.

If you specify ``--dry-run`` before a task, then the action of that task will
not actually be carried out, although logging output will be displayed as if
it were. For example, you could run ``paver --dry-run tinymce_zip`` to see what
files would be added to the ``tinymce.zip`` file, etc.
"""

from cStringIO import StringIO
import shutil
import sys
import zipfile

import paver
import paver.easy
import paver.tasks
from paver.easy import *
from paver.path import path


# Paver comes with Jason Orendorff's 'path' module; this makes path
# manipulation easy and far more readable.
PROJECT_DIR = path(__file__).dirname().abspath()


# Set some default options. Having the options at the top of the file cleans
# the whole thing up and makes the behaviour a lot more configurable.
options(
  build = Bunch(
    project_dir = PROJECT_DIR,
    app_build = PROJECT_DIR / 'build',
    app_folder = PROJECT_DIR / 'app',
    overrides_folder = PROJECT_DIR / 'overrides',
    overrides_dirs = ['soc', 'soc/models'],
    overrides_files = ['soc/models/universities.py'],
    app_files = ['app.yaml', 'index.yaml', 'queue.yaml', 'cron.yaml',
                 'mapreduce.yaml', 'main.py', 'settings.py', 'urls.py',
                 'gae_django.py', 'profiler.py', 'appengine_config.py'],
    app_dirs =  ["soc", "feedparser", "djangoforms",
                 "jquery.min", "ranklist", "shell", "json.min", "jlinq",
                 "modernizr.min", "html5lib", "LABjs.min", "gviz","webmaster",
                 "gdata", "atom", "mapreduce"],
    css_dirs = ["soc/content/css/v2/gsoc/", "soc/content/css/v2/gci"],
    css_files = {
        "jquery-ui/jquery.ui.merged.css": [
            "jquery-ui/jquery.ui.core.css",
            "jquery-ui/jquery.ui.resizable.css",
            "jquery-ui/jquery.ui.selectable.css",
            "jquery-ui/jquery.ui.accordion.css",
            "jquery-ui/jquery.ui.autocomplete.css",
            "jquery-ui/jquery.ui.button.css",
            "jquery-ui/jquery.ui.dialog.css",
            "jquery-ui/jquery.ui.slider.css",
            "jquery-ui/jquery.ui.tabs.css",
            "jquery-ui/jquery.ui.datepicker.css",
            "jquery-ui/jquery.ui.progressbar.css",
            "jquery-ui/jquery.ui.theme.css"
        ],
    },
    zip_files = ['tiny_mce.zip'],
    skip_pylint = False,
    skip_closure = False,
  )
)

# The second call to options allows us to re-use some of the constants defined
# in the first call.
options(
  clean_build = options.build,
  tinymce_zip = options.build,

  pylint = Bunch(
    check_modules = [
        'soc/models',
        'soc/modules/gsoc/models',
        'soc/modules/gci/models',
        #'soc/logic',
        #'soc/modules/gsoc/logic',
        #'soc/modules/gci/logic',
        #'soc/views/helper',
        #'soc/modules/gsoc/views/helper',
        #'soc/modules/gci/views/helper',
        'reflistprop',
        'settings.py',
        'urls.py',
        'main.py',
    ],
    quiet = False,
    quiet_args = [
      '--reports=no',
      '--errors-only',
      '--disable=E1103',
    ],
    pylint_args = [],
    with_module = None,
    ignore = False,
    **options.build
  ),

  closure = Bunch(
    js_filter = None,
    js_dir = None,
    js_dirs = ["soc/content/js", "jquery", "jlinq",
               "modernizr", "json", "LABjs"],
    closure_bin = PROJECT_DIR / "thirdparty/closure/compiler.jar",
    no_optimize = ["jquery-spin-1.1.1.js", "jquery-jqgrid.base.js"],
    **options.build
  )
)


# Utility functions

def tinymce_zip_files(tiny_mce_dir):
  """Yields each filename which should go into ``tiny_mce.zip``."""
  for filename in tiny_mce_dir.walkfiles():
    if '.svn' in filename.splitall():
      continue

    paver.tasks.environment.info(
        '%-4stiny_mce.zip <- %s', '', filename)
    arcname = tiny_mce_dir.relpathto(filename)
    yield filename, arcname


def write_zip_file(zip_file_handle, files):
  if paver.tasks.environment.dry_run:
    for args in files:
      pass
    return
  zip_file = zipfile.ZipFile(zip_file_handle, mode='w')
  for args in files:
    zip_file.write(*args)
  zip_file.close()


def symlink(target, link_name):
  if hasattr(target, 'symlink'):
    target.symlink(link_name)
  else:
    # If we are on a platform where symlinks are not supported (such as
    # Windows), simply copy the files across.
    target.copy(link_name)


# Tasks


@task
@cmdopts([
    ('app-folder=', 'a', 'App folder directory (default /app)'),
    ('pylint-command=', 'c', 'Specify a custom pylint executable'),
    ('with-module=', 'w', 'Include a specific module'),
    ('quiet', 'q', 'Disables a lot of the pylint output'),
    ('ignore', 'i', 'Ignore PyLint errors')
])
def pylint(options):
  """Check the source code using PyLint."""
  from pylint import lint

  # Initial command.
  arguments = []

  if options.quiet:
    arguments.extend(options.quiet_args)
  if 'pylint_args' in options:
    arguments.extend(list(options.pylint_args))

  # Add the list of paths containing the modules to check using PyLint.
  if options.with_module:
    arguments.append(options.with_module)
  else:
    arguments.extend(
        str(options.app_folder / module) for module in options.check_modules)


  # By placing run_pylint into its own function, it allows us to do dry runs
  # without actually running PyLint.
  def run_pylint():
    # Add app folder to path.
    sys.path.insert(0, options.app_folder.abspath())
    # Add google_appengine directory to path.
    path = options.project_dir.abspath() / 'thirdparty' / 'google_appengine'
    sys.path.insert(0, path)
    
    # Specify PyLint RC file.
    path = options.project_dir.abspath() / 'scripts' / 'pylint' / 'pylintrc'
    arguments.append('--rcfile=' + path)

    # `lint.Run.__init__` runs the PyLint command.
    try:
      lint.Run(arguments)
    # PyLint will `sys.exit()` when it has finished, so we need to catch
    # the exception and process it accordingly.
    except SystemExit, exc:
      return_code = exc.args[0]
      if return_code != 0 and (not options.pylint.ignore):
        raise paver.tasks.BuildFailure(
            'PyLint finished with a non-zero exit code')

  return dry('pylint ' + ' '.join(arguments), run_pylint)


@task
@cmdopts([
    ('app-build=', 'b', 'App build directory (default /build)'),
    ('app-folder=', 'a', 'App folder directory (default /app)'),
    ('skip-pylint', 's', 'Skip PyLint checker'),
    ('ignore-pylint', 'i', 'Ignore results of PyLint (but run it anyway)'),
    ('quiet-pylint', 'q', 'Make PyLint run quietly'),
])
def build(options):
  """Build the project."""
  # If `--skip-pylint` is not provided, run PyLint.
  if not options.skip_pylint:
    # If `--ignore-pylint` is provided, act as if `paver pylint --ignore`
    # was run. Likewise for `--quiet-pylint`.
    if options.get('ignore_pylint', False):
      options.pylint.ignore = True
    if options.get('quiet_pylint', False):
      options.pylint.quiet = True
    pylint(options)

  # Compile the css files into one
  build_css(options)

  # Clean old generated zip files from the app folder.
  clean_zip(options)

  # Clean the App build directory by removing and re-creating it.
  clean_build(options)

  # Build the tiny_mce.zip file.
  tinymce_zip(options)

  # Make the necessary symlinks between the app and build directories.
  build_symlinks(options)

  # Handle overrides
  overrides(options)


@task
@cmdopts([
    ('app-build=', 'b', 'App build directory (default /build)'),
    ('app-folder=', 'a', 'App folder directory (default /app)'),
])
def build_symlinks(options):
  """Build symlinks between the app and build folders."""
  # Create the symbolic links from the app folder to the build folder.
  for filename in options.app_files + options.app_dirs + options.zip_files:
    # The `symlink()` function handles discrepancies between platforms.
    target = path(options.app_folder) / filename
    link = path(options.app_build) / filename
    dry('%-4s%-20s <- %s' % ('', target, link),
        lambda: symlink(target, link.abspath()))


@task
def build_css(options):
  """Compiles the css files into one."""

  for css_dir in options.css_dirs:
    for target, components in options.css_files.iteritems():
      target = options.app_folder / css_dir / target
      f = target.open('w')

      for component in components:
        source = options.app_folder / css_dir / component
        dry("cat %s >> %s" % (source, target),
            lambda: shutil.copyfileobj(source.open('r'), f))
      f.close()


@task
@cmdopts([
    ('app-build=', 'b', 'App build directory (default /build)'),
])
def clean_build(options):
  """Clean the build folder."""
  # Not checking this could cause an error when trying to remove a
  # non-existent file.
  if path(options.app_build).exists():
    path(options.app_build).rmtree()
  path(options.app_build).makedirs_p()


@task
@cmdopts([
    ('app-folder=', 'a', 'App folder directory (default /app)'),
])
def clean_zip(options):
  """Remove all the generated zip files from the app folder."""
  for zip_file in options.zip_files:
    zip_path = path(options.app_folder) / zip_file
    if zip_path.exists():
      zip_path.remove()


@task
@cmdopts([
    ('app-folder=', 'a', 'App folder directory (default /app)'),
])
def tinymce_zip(options):
  """Create the zip file containing TinyMCE."""
  tinymce_dir = path(options.app_folder) / 'tiny_mce'
  tinymce_zip_filename = path(options.app_folder) / 'tiny_mce.zip'
  if paver.tasks.environment.dry_run:
    tinymce_zip_fp = StringIO()
  else:
    # Ensure the parent directories exist.
    tinymce_zip_filename.dirname().makedirs_p()
    tinymce_zip_fp = open(tinymce_zip_filename, mode='w')

  try:
    write_zip_file(tinymce_zip_fp, tinymce_zip_files(tinymce_dir))
  except Exception, exc:
    tinymce_zip_fp.close()
    tinymce_zip_filename.remove()
    raise paver.tasks.BuildFailure(
        'Error occurred creating tinymce.zip: %r' % (exc,))
  finally:
    if not tinymce_zip_fp.closed:
      tinymce_zip_fp.close()

def run_closure(f):
  """Runs the closure compiler over one JS file"""

  tmp = f + ".tmp.js"
  f.move(tmp)

  try:
    sh("java -jar '%s' --js='%s' > '%s'" % (options.closure_bin, tmp, f))
  except BuildFailure, e:
    paver.tasks.environment.error(
        "%s minimization failed, copying plain file", f)
    tmp.copy(f)

  tmp.remove()

@task
@cmdopts([
    ('app-folder=', 'a', 'App folder directory (default /app)'),
    ('js-dir=', 'j', 'JS directory to minimize, relative to /app'),
    ('js-filter=', 'f', 'Minimize files matching this regex, default "*.js"'),
])
def closure(options):
  """Runs the closure compiler over the JS files."""

  if options.js_dir:
    dirs = [options.app_folder / options.js_dir]
  else:
    dirs = [options.app_folder / i for i in options.js_dirs]
  old_size = 0
  new_size = 0

  js_filter = options.js_filter if options.js_filter else "*.js"

  for js_dir in dirs:
    min_dir = js_dir + ".min"

    if not options.js_filter:
      min_dir.rmtree()

    js_dir.copytree(min_dir)

    for f in min_dir.walkfiles(js_filter):
      if f.name in options.no_optimize:
        paver.tasks.environment.info(
            '%-4sCLOSURE: Skipping %s', '', f)
        continue

      paver.tasks.environment.info(
          '%-4sCLOSURE: Processing %s', '', f)

      old_size += f.size

      run_closure(f)

      new_size += f.size

  rate = new_size*100 / old_size
  paver.tasks.environment.info(
      "%-4sCLOSURE: Source file sizes: %s, Dest file sizes: %s, Rate: %s",
      '', old_size, new_size, rate)


@task
def overrides(options):
  """Copies files from the overrides structure to the build directory.
  """
  for path in options.overrides_dirs:
    target = options.app_build / path
    unroll_symlink(target)
  for path in options.overrides_files:
    target = options.overrides_folder / path
    if not target.exists():
      continue
    if not target.isfile():
      paver.tasks.environment.info(
          "target '%s' is not a file", target)
      continue
    to = options.app_build / path
    to.remove()
    target.symlink(to)


def unroll_symlink(target):
  """Unrolls a symlink.

  Does the following if target is a directory symlink:
  - removes the symlink
  - creates a directory with the same name
  - populates it with symlinks to individual files

  Otherwise does nothing.
  """
  if not target.exists():
    paver.tasks.environment.info(
        "target '%s' does not exist", target)
    return
  if not target.isdir():
    paver.tasks.environment.info(
        "target '%s' is not a directory", target)
    return
  if not target.islink():
    paver.tasks.environment.info(
        "target '%s' is not a symlink", target)
    return

  deref = target.readlinkabs()
  target.remove()
  target.mkdir()

  contents = deref.listdir()

  for path in contents:
    path.symlink(target / path.name)