blob: e3dec773b828ba93449845e9ae8bacaf203bb7e0 [file] [log] [blame]
# -*- coding: utf-8 -*-
"""
Example Usage
=============
The following commands can be run from the root directory of the Mercurial
repo. To run ``paver``, however, you'll need to do ``easy_install Paver``.
Most of the following commands accept other arguments; see ``command --help``
for more information, or ``paver help`` for a list of all the valid commands.
``paver build``
Builds the project. This essentially just runs a bunch of other tasks,
like ``pylint`` and ``tinymce_zip``, etc.
``paver pylint``
Runs PyLint on the project.
``paver tinymce_zip``
Builds the TinyMCE zip file.
If you specify ``--dry-run`` before a task, then the action of that task will
not actually be carried out, although logging output will be displayed as if
it were. For example, you could run ``paver --dry-run tinymce_zip`` to see what
files would be added to the ``tinymce.zip`` file, etc.
"""
import cStringIO
import os
import shutil
import sys
import zipfile
from google.appengine.ext import testbed
from epydoc import cli
from paver import easy
from paver import path
from paver import tasks
from pylint import lint
# Paver comes with Jason Orendorff's 'path' module; this makes path
# manipulation easy and far more readable.
PROJECT_DIR = path.path(__file__).dirname().abspath()
REPORTS_DIR = PROJECT_DIR / 'reports'
JS_DIRS = ['soc/content/js']
COPY_DIRS = ['soc/content/css']
DONT_COPY_DIRS = [] # subset of COPY_DIRS to exclude
APP_FILES = [
'app.yaml',
'index.yaml',
'queue.yaml',
'cron.yaml',
'mapreduce.yaml',
'main.py',
'settings.py',
'urls.py',
'gae_django.py',
'profiler.py',
'appengine_config.py',
]
APP_DIRS = [
'melange',
'soc',
'feedparser',
'djangoforms',
'ranklist',
'shell',
'html5lib',
'gviz',
'webmaster',
'mapreduce',
'summerofcode',
'codein',
]
# These files are copied from /thirdparty to /build
THIRDPARTY_DIRS = [
'apiclient',
'httplib2',
'oauth2client',
'uritemplate',
'cloudstorage',
]
CSS_FILES = {
'thirdparty/jquery-ui/jquery.ui.merged.css': [
'thirdparty/jquery-ui/jquery.ui.core.css',
'thirdparty/jquery-ui/jquery.ui.resizable.css',
'thirdparty/jquery-ui/jquery.ui.selectable.css',
'thirdparty/jquery-ui/jquery.ui.accordion.css',
'thirdparty/jquery-ui/jquery.ui.autocomplete.css',
'thirdparty/jquery-ui/jquery.ui.button.css',
'thirdparty/jquery-ui/jquery.ui.dialog.css',
'thirdparty/jquery-ui/jquery.ui.slider.css',
'thirdparty/jquery-ui/jquery.ui.tabs.css',
'thirdparty/jquery-ui/jquery.ui.datepicker.css',
'thirdparty/jquery-ui/jquery.ui.progressbar.css',
'thirdparty/jquery-ui/jquery.ui.theme.css',
],
}
CSS_DIRS = ['soc/content/css/gsoc/', 'soc/content/css/gci']
ZIP_FILES = ['tiny_mce.zip']
DOCS_CONFIG = PROJECT_DIR / 'docs.config'
DOCS_OUTPUT = REPORTS_DIR / 'docs'
# TODO(nathaniel): Get rid of all the "overrides" stuff as part of
# finishing https://code.google.com/p/soc/issues/detail?id=1560.
OVERRIDES_FOLDER = PROJECT_DIR / 'overrides'
OVERRIDES_DIRS = ['soc', 'soc/models', 'soc/content']
OVERRIDES_FILES = ['soc/models/universities.py']
BUILD_BUNCH = easy.Bunch(
project_dir=PROJECT_DIR,
app_files=APP_FILES,
app_dirs=APP_DIRS,
app_build=PROJECT_DIR / 'build',
app_folder=PROJECT_DIR / 'app',
thirdparty_dirs=THIRDPARTY_DIRS,
thirdparty_folder=PROJECT_DIR / 'thirdparty',
css_dirs=CSS_DIRS,
css_files=CSS_FILES,
zip_files=ZIP_FILES,
docs_config=DOCS_CONFIG,
docs_output=DOCS_OUTPUT,
copy_dirs=COPY_DIRS,
dont_copy_dirs=DONT_COPY_DIRS,
overrides_folder=OVERRIDES_FOLDER,
overrides_dirs=OVERRIDES_DIRS,
overrides_files=OVERRIDES_FILES,
skip_docs=False,
skip_pylint=False)
PYLINT_APP_FOLDER_MODULES = [
'codein',
'melange',
'soc',
'summerofcode',
'settings.py',
'urls.py',
'main.py',
]
PYLINT_TESTS_FOLDER = PROJECT_DIR / 'tests'
PYLINT_PROJECT_FOLDER_MODULES = ['pavement.py', 'setup.py']
PYLINT_VERBOSE_ARGS = [
# In the rcfile(pylintrc) errors-only option is set. This is to enable
# other messages as well.
# R and C modules are just too chatty, we can however turn a few of the
# more useful ones on explicitly.
'--enable=W,F',
'--reports=yes',
# We may want to enable these in the future
'--disable=protected-access,attribute-defined-outside-init',
# TODO(nathaniel): fix all occurences and enable this
'--disable=abstract-method',
# These are just plain useless, we don't ever want to these
'--disable=fixme,unused-argument,star-args,bad-builtin,locally-disabled',
# These are somewhat debatable, but not realistic for Melange
'--disable=no-init,super-init-not-called',
# TODO(nathaniel): fix all occurences and move this to pylintrc file.
'--enable=line-too-long',
]
PYLINT_BUNCH = easy.Bunch(
app_folder_modules=PYLINT_APP_FOLDER_MODULES,
project_folder_modules=PYLINT_PROJECT_FOLDER_MODULES,
tests_folder=PYLINT_TESTS_FOLDER,
verbose=False,
verbose_args=PYLINT_VERBOSE_ARGS,
pylint_args=[],
with_module=None,
ignore=False,
**BUILD_BUNCH)
# Install the option bunches.
easy.options(
build=BUILD_BUNCH,
clean_build=BUILD_BUNCH,
tinymce_zip=BUILD_BUNCH,
pylint=PYLINT_BUNCH)
# Utility functions
def tinymce_zip_files(tiny_mce_dir):
"""Yields each filename which should go into ``tiny_mce.zip``."""
for filename in tiny_mce_dir.walkfiles():
if '.svn' in filename.splitall():
continue
tasks.environment.info('%-4stiny_mce.zip <- %s', '', filename)
arcname = tiny_mce_dir.relpathto(filename)
yield filename, arcname
def write_zip_file(zip_file_handle, files):
if tasks.environment.dry_run:
for args in files:
pass
return
zip_file = zipfile.ZipFile(zip_file_handle, mode='w')
for args in files:
zip_file.write(*args)
zip_file.close()
def symlink(target, link_name):
if hasattr(target, 'symlink'):
target.symlink(link_name)
else:
# If we are on a platform where symlinks are not supported (such as
# Windows), simply copy the files across.
target.copy(link_name)
# Tasks
@easy.task
@easy.cmdopts([
('app-folder=', 'a', 'App folder directory (default /app)'),
('pylint-command=', 'c', 'Specify a custom pylint executable'),
('with-module=', 'w', 'Include a specific module'),
('verbose', 'v', 'Enables a lot of the noisy pylint output'),
('ignore', 'i', 'Ignore PyLint errors')
])
def pylint(options):
"""Check the source code using PyLint."""
# Initial command.
arguments = []
if options.verbose:
arguments.extend(options.verbose_args)
if 'pylint_args' in options:
arguments.extend(list(options.pylint_args))
# Add the list of paths containing the modules to check using PyLint.
if options.with_module:
arguments.append(options.with_module)
else:
arguments.extend(str(options.app_folder / module)
for module in options.app_folder_modules)
arguments.extend(str(options.project_dir / module)
for module in options.project_folder_modules)
# We should lint everything in the tests folder.
arguments.append(str(options.tests_folder))
# By placing run_pylint into its own function, it allows us to do dry runs
# without actually running PyLint.
def run_pylint():
# Add app folder to path.
sys.path.insert(0, options.app_folder.abspath())
# Add google_appengine directory to path.
gae_path = options.project_dir.abspath() / 'thirdparty' / 'google_appengine'
sys.path.insert(0, gae_path)
# Specify PyLint RC file.
pylint_path = options.project_dir.abspath() / 'pylintrc'
arguments.append('--rcfile=' + pylint_path)
# `lint.Run.__init__` runs the PyLint command.
try:
lint.Run(arguments)
# PyLint will `sys.exit()` when it has finished, so we need to catch
# the exception and process it accordingly.
except SystemExit, exc:
return_code = exc.args[0]
if return_code != 0 and (not options.pylint.ignore):
raise tasks.BuildFailure(
'PyLint finished with a non-zero exit code: %d' % return_code)
return easy.dry('pylint ' + ' '.join(arguments), run_pylint)
@easy.task
@easy.cmdopts([
('app-build=', 'b', 'App build directory (default /build)'),
('app-folder=', 'a', 'App folder directory (default /app)'),
('skip-pylint', 's', 'Skip PyLint checker'),
('skip-docs', '', 'Skip documentation creation'),
('ignore-pylint', 'i', 'Ignore results of PyLint (but run it anyway)'),
('verbose-pylint', 'v', 'Make PyLint run verbosely'),
])
def build(options):
"""Build the project."""
# If `--skip-pylint` is not provided, run PyLint.
if not options.skip_pylint:
# If `--ignore-pylint` is provided, act as if `paver pylint --ignore`
# was run. Likewise for `--verbose-pylint`.
if options.get('ignore_pylint', False):
options.pylint.ignore = True
if options.get('verbose_pylint', False):
options.pylint.verbose = True
pylint(options)
# Compile the css files into one
build_css(options)
# Clean old generated zip files from the app folder.
clean_zip(options)
# Clean the App build directory by removing and re-creating it.
clean_build(options)
# Build the tiny_mce.zip file.
tinymce_zip(options)
# Make the necessary symlinks between the app and build directories.
build_symlinks(options)
# Handle overrides
overrides(options)
# Handle deep overrides (copy)
deep_overrides(options)
# Run grunt for production
run_grunt(options)
# Builds documentation for the project
if not options.skip_docs:
build_docs(options)
@easy.task
def run_grunt(options):
"""Run Grunt for build"""
easy.sh('bin/grunt build -f')
easy.sh('bin/grunt build_dev -f')
def symlink_file(source_folder, target_folder, filename):
# The `symlink()` function handles discrepancies between platforms.
source_file = path.path(source_folder) / filename
target_file = path.path(target_folder) / filename
easy.dry('%-4s%-20s <- %s' % ('', source_file, target_file),
lambda: symlink(source_file, target_file.abspath()))
@easy.task
@easy.cmdopts([
('app-build=', 'b', 'App build directory (default /build)'),
('app-folder=', 'a', 'App folder directory (default /app)'),
])
def build_symlinks(options):
"""Build symlinks between the app and build folders."""
# Create the symbolic links from the app folder to the build folder.
for filename in options.app_files + options.app_dirs + options.zip_files:
symlink_file(options.app_folder, options.app_build, filename)
for filename in options.thirdparty_dirs:
symlink_file(options.thirdparty_folder, options.app_build, filename)
@easy.task
def build_css(options):
"""Compiles the css files into one."""
for css_dir in options.css_dirs:
for target, components in options.css_files.iteritems():
target = options.app_folder / css_dir / target
with target.open('w') as target_file:
for component in components:
source = options.app_folder / css_dir / component
easy.dry(
"cat %s >> %s" % (source, target),
lambda: shutil.copyfileobj(source.open('r'), target_file))
@easy.task
@easy.cmdopts([
('app-build=', 'b', 'App build directory (default /build)'),
])
def clean_build(options):
"""Clean the build folder."""
# Not checking this could cause an error when trying to remove a
# non-existent file.
if path.path(options.app_build).exists():
path.path(options.app_build).rmtree()
path.path(options.app_build).makedirs_p()
@easy.task
@easy.cmdopts([
('app-folder=', 'a', 'App folder directory (default /app)'),
])
def clean_zip(options):
"""Remove all the generated zip files from the app folder."""
for zip_file in options.zip_files:
zip_path = path.path(options.app_folder) / zip_file
if zip_path.exists():
zip_path.remove()
@easy.task
@easy.cmdopts([
('app-folder=', 'a', 'App folder directory (default /app)'),
])
def tinymce_zip(options):
"""Create the zip file containing TinyMCE.
This zip file then gets served with google.appengine.ext.zipserve.
(Configured in app.yaml).
Why? A long time ago, in a galaxy far away, App Engine had a limit
on the number of files that could be in an app, and Melange was
bumping up against that limit.
This isn't strictly necessary anymore, but it works and there's no
reason to remove it. It probably makes the release process slightly
faster, because fewer files need to be uploaded.
"""
tinymce_dir = path.path(
options.app_folder) / 'soc/content/js/thirdparty/tiny_mce'
tinymce_zip_filename = path.path(options.app_folder) / 'tiny_mce.zip'
if tasks.environment.dry_run:
tinymce_zip_fp = cStringIO.StringIO()
else:
# Ensure the parent directories exist.
tinymce_zip_filename.dirname().makedirs_p()
tinymce_zip_fp = open(tinymce_zip_filename, mode='w')
try:
write_zip_file(tinymce_zip_fp, tinymce_zip_files(tinymce_dir))
except Exception, exc:
tinymce_zip_fp.close()
tinymce_zip_filename.remove()
raise tasks.BuildFailure(
'Error occurred creating tinymce.zip: %r' % (exc,))
finally:
if not tinymce_zip_fp.closed:
tinymce_zip_fp.close()
@easy.task
@easy.cmdopts([
('docs-output=', '', 'Output directory for documentation'),
('docs-config=', '', 'Configuration file for documentation'),
])
def build_docs(options):
"""Builds documentation for the project."""
# Epydoc smartly makes its own output directory if the output
# directory doesn't exist, but is not so smart that it will
# recursively create the parents of the output directory if they
# don't exist. Since we have the output directory path here we
# might as well create it rather than parsing out just the parents
# and leaving the directory itself to epydoc.
if not os.path.exists(options.docs_output):
os.makedirs(options.docs_output)
# NOTE(nathaniel): Epydoc actually imports modules during analysis,
# Melange's modules in turn import App Engine modules, and App Engine
# modules complain if the right Django and App Engine settings aren't
# in place at import time. Consequently, we must mutate the current
# environment to be that of an App Engine test before we can build
# Melange's documentation.
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
os.environ['SERVER_SOFTWARE'] = 'build'
appengine_testbed = testbed.Testbed()
appengine_testbed.activate()
appengine_testbed.init_datastore_v3_stub()
# NOTE(nathaniel): Deriving the options to pass to (epydoc.)cli.main this
# way is horsehockey, but epydoc doesn't actually expose a proper API.
stored_actual_argv = sys.argv
sys.argv = [
'unused_fake_executuable',
'--config=%s' % options.docs_config,
'--output=%s' % options.docs_output,
]
epydoc_options, epydoc_names = cli.parse_arguments()
sys.argv = stored_actual_argv
# NOTE(nathaniel): As of 13 January 2014 this call emits two false
# positive "Bad argument - expected name or tuple" errors. See
# https://sourceforge.net/p/epydoc/bugs/363/ for progress.
cli.main(epydoc_options, epydoc_names)
@easy.task
def deep_overrides(options):
"""Copies files from the copy structure to the build directory."""
dirs = [options.app_folder / i for i in options.copy_dirs]
dont_copy_dirs = [options.app_folder / i for i in options.dont_copy_dirs]
for source_dir in dirs:
dest_dir = options.app_build / options.app_folder.relpathto(source_dir)
dest_dir.remove()
source_dir.copytree(dest_dir)
for remove_dir in dont_copy_dirs:
dest_dir = options.app_build / options.app_folder.relpathto(remove_dir)
dest_dir.rmtree()
@easy.task
def overrides(options):
"""Copies files from the overrides structure to the build directory."""
for override_path in options.overrides_dirs:
target = options.app_build / override_path
unroll_symlink(target)
for override_path in options.overrides_files:
target = options.overrides_folder / override_path
if not target.exists():
continue
if not target.isfile():
tasks.environment.info('target "%s" is not a file', target)
continue
to = options.app_build / override_path
to.remove()
target.symlink(to)
def unroll_symlink(target):
"""Unrolls a symlink.
Does the following if target is a directory symlink:
- removes the symlink
- creates a directory with the same name
- populates it with symlinks to individual files
Otherwise does nothing.
"""
if not target.exists():
tasks.environment.info('target "%s" does not exist', target)
return
if not target.isdir():
tasks.environment.info('target "%s" is not a directory', target)
return
if not target.islink():
tasks.environment.info('target "%s" is not a symlink', target)
return
deref = target.readlinkabs()
target.remove()
target.mkdir()
contents = deref.listdir()
for symlink_path in contents:
symlink_path.symlink(target / symlink_path.name)