blob: 1ff6c7913b9cb42c542d494682ea3bb2448d54ec [file] [log] [blame]
# -*- coding: utf-8 -*-
"""
Example Usage
=============
The following commands can be run from the root directory of the Mercurial
repo. To run ``paver``, however, you'll need to do ``easy_install Paver``.
Most of the following commands accept other arguments; see ``command --help``
for more information, or ``paver help`` for a list of all the valid commands.
``paver build``
Builds the project. This essentially just runs a bunch of other tasks,
like ``tinymce_zip``, etc.
``paver tinymce_zip``
Builds the TinyMCE zip file.
If you specify ``--dry-run`` before a task, then the action of that task will
not actually be carried out, although logging output will be displayed as if
it were. For example, you could run ``paver --dry-run tinymce_zip`` to see what
files would be added to the ``tinymce.zip`` file, etc.
"""
import cStringIO
import os
import shutil
import sys
import zipfile
from google.appengine.ext import testbed
from epydoc import cli
from paver import easy
from paver import path
from paver import tasks
# Paver comes with Jason Orendorff's 'path' module; this makes path
# manipulation easy and far more readable.
PROJECT_DIR = path.path(__file__).dirname().abspath()
REPORTS_DIR = PROJECT_DIR / 'reports'
JS_DIRS = ['soc/content/js']
COPY_DIRS = ['soc/content/css']
DONT_COPY_DIRS = [] # subset of COPY_DIRS to exclude
APP_FILES = [
'app.yaml',
'index.yaml',
'queue.yaml',
'cron.yaml',
'mapreduce.yaml',
'main.py',
'settings.py',
'urls.py',
'profiler.py',
'appengine_config.py',
]
APP_DIRS = [
'melange',
'soc',
'feedparser',
'djangoforms',
'ranklist',
'shell',
'html5lib',
'gviz',
'webmaster',
'summerofcode',
'codein',
]
# These files are copied from /thirdparty to /build
THIRDPARTY_DIRS = [
'apiclient',
'enum',
'httplib2',
'oauth2client',
'uritemplate',
'cloudstorage',
'mapreduce',
'pipeline',
'graphy',
'simplejson',
]
CSS_FILES = {
'thirdparty/jquery-ui/jquery.ui.merged.css': [
'thirdparty/jquery-ui/jquery.ui.core.css',
'thirdparty/jquery-ui/jquery.ui.resizable.css',
'thirdparty/jquery-ui/jquery.ui.selectable.css',
'thirdparty/jquery-ui/jquery.ui.accordion.css',
'thirdparty/jquery-ui/jquery.ui.autocomplete.css',
'thirdparty/jquery-ui/jquery.ui.button.css',
'thirdparty/jquery-ui/jquery.ui.dialog.css',
'thirdparty/jquery-ui/jquery.ui.slider.css',
'thirdparty/jquery-ui/jquery.ui.tabs.css',
'thirdparty/jquery-ui/jquery.ui.datepicker.css',
'thirdparty/jquery-ui/jquery.ui.progressbar.css',
'thirdparty/jquery-ui/jquery.ui.theme.css',
],
}
CSS_DIRS = ['soc/content/css/gsoc/', 'soc/content/css/gci']
ZIP_FILES = ['tiny_mce.zip']
DOCS_CONFIG = PROJECT_DIR / 'docs.config'
DOCS_OUTPUT = REPORTS_DIR / 'docs'
# TODO(nathaniel): Get rid of all the "overrides" stuff as part of
# finishing https://code.google.com/p/soc/issues/detail?id=1560.
OVERRIDES_FOLDER = PROJECT_DIR / 'overrides'
OVERRIDES_DIRS = ['soc', 'soc/models', 'soc/content']
OVERRIDES_FILES = ['soc/models/universities.py']
BUILD_BUNCH = easy.Bunch(
project_dir=PROJECT_DIR,
app_files=APP_FILES,
app_dirs=APP_DIRS,
app_build=PROJECT_DIR / 'build',
app_folder=PROJECT_DIR / 'app',
thirdparty_dirs=THIRDPARTY_DIRS,
thirdparty_folder=PROJECT_DIR / 'thirdparty',
css_dirs=CSS_DIRS,
css_files=CSS_FILES,
zip_files=ZIP_FILES,
docs_config=DOCS_CONFIG,
docs_output=DOCS_OUTPUT,
copy_dirs=COPY_DIRS,
dont_copy_dirs=DONT_COPY_DIRS,
overrides_folder=OVERRIDES_FOLDER,
overrides_dirs=OVERRIDES_DIRS,
overrides_files=OVERRIDES_FILES,
skip_docs=False)
# Install the option bunches.
easy.options(
build=BUILD_BUNCH,
clean_build=BUILD_BUNCH,
tinymce_zip=BUILD_BUNCH,
)
# Utility functions
def tinymce_zip_files(tiny_mce_dir):
"""Yields each filename which should go into ``tiny_mce.zip``."""
for filename in tiny_mce_dir.walkfiles():
if '.svn' in filename.splitall():
continue
tasks.environment.info('%-4stiny_mce.zip <- %s', '', filename)
arcname = tiny_mce_dir.relpathto(filename)
yield filename, arcname
def write_zip_file(zip_file_handle, files):
if tasks.environment.dry_run:
for args in files:
pass
return
zip_file = zipfile.ZipFile(zip_file_handle, mode='w')
for args in files:
zip_file.write(*args)
zip_file.close()
def symlink(target, link_name):
if hasattr(target, 'symlink'):
target.symlink(link_name)
else:
# If we are on a platform where symlinks are not supported (such as
# Windows), simply copy the files across.
target.copy(link_name)
# Tasks
@easy.task
@easy.cmdopts([
('app-build=', 'b', 'App build directory (default /build)'),
('app-folder=', 'a', 'App folder directory (default /app)'),
('skip-docs', '', 'Skip documentation creation'),
])
def build(options):
"""Build the project."""
# Compile the css files into one
build_css(options)
# Clean old generated zip files from the app folder.
clean_zip(options)
# Clean the App build directory by removing and re-creating it.
clean_build(options)
# Build the tiny_mce.zip file.
tinymce_zip(options)
# Make the necessary symlinks between the app and build directories.
build_symlinks(options)
# Handle overrides
overrides(options)
# Handle deep overrides (copy)
deep_overrides(options)
# Run grunt for production
run_grunt(options)
# Builds documentation for the project
if not options.skip_docs:
build_docs(options)
@easy.task
def run_grunt(options):
"""Run Grunt for build"""
easy.sh('bin/grunt build -f')
easy.sh('bin/grunt build_dev -f')
def symlink_file(source_folder, target_folder, filename, target_filename=None):
if not target_filename:
target_filename = filename
# The `symlink()` function handles discrepancies between platforms.
source_file = path.path(source_folder) / filename
target_file = path.path(target_folder) / target_filename
if (os.path.exists(target_file) and
os.readlink(target_file) == source_file):
easy.info('symlink for %s already points to the right place, skipping' % (
target_file))
else:
easy.dry('%-4s%-20s <- %s' % ('', source_file, target_file),
lambda: symlink(source_file, target_file.abspath()))
@easy.task
@easy.cmdopts([
('app-build=', 'b', 'App build directory (default /build)'),
('app-folder=', 'a', 'App folder directory (default /app)'),
])
def build_symlinks(options):
"""Build symlinks between the app and build folders."""
# Create the symbolic links from the app folder to the build folder.
for filename in options.app_files + options.app_dirs + options.zip_files:
symlink_file(options.app_folder, options.app_build, filename)
for filename in options.thirdparty_dirs:
symlink_file(options.thirdparty_folder, options.app_build, filename)
# thirdparty/js -> build/soc/content/js/thirdparty
# required to serve thirdparty JavaScript files to clients
thirdparty_js_dir = options.app_folder / 'soc' / 'content' / 'js'
symlink_file(options.thirdparty_folder, thirdparty_js_dir, 'js', 'thirdparty')
@easy.task
def build_css(options):
"""Compiles the css files into one."""
for css_dir in options.css_dirs:
for target, components in options.css_files.iteritems():
target = options.app_folder / css_dir / target
with target.open('w') as target_file:
for component in components:
source = options.app_folder / css_dir / component
easy.dry(
"cat %s >> %s" % (source, target),
lambda: shutil.copyfileobj(source.open('r'), target_file))
@easy.task
@easy.cmdopts([
('app-build=', 'b', 'App build directory (default /build)'),
])
def clean_build(options):
"""Clean the build folder."""
# Not checking this could cause an error when trying to remove a
# non-existent file.
if path.path(options.app_build).exists():
path.path(options.app_build).rmtree()
path.path(options.app_build).makedirs_p()
@easy.task
@easy.cmdopts([
('app-folder=', 'a', 'App folder directory (default /app)'),
])
def clean_zip(options):
"""Remove all the generated zip files from the app folder."""
for zip_file in options.zip_files:
zip_path = path.path(options.app_folder) / zip_file
if zip_path.exists():
zip_path.remove()
@easy.task
@easy.cmdopts([
('app-folder=', 'a', 'App folder directory (default /app)'),
])
def tinymce_zip(options):
"""Create the zip file containing TinyMCE.
This zip file then gets served with google.appengine.ext.zipserve.
(Configured in app.yaml).
Why? A long time ago, in a galaxy far away, App Engine had a limit
on the number of files that could be in an app, and Melange was
bumping up against that limit.
This isn't strictly necessary anymore, but it works and there's no
reason to remove it. It probably makes the release process slightly
faster, because fewer files need to be uploaded.
"""
tinymce_dir = path.path(
options.project_dir) / 'thirdparty/js/tiny_mce'
tinymce_zip_filename = path.path(options.app_folder) / 'tiny_mce.zip'
if tasks.environment.dry_run:
tinymce_zip_fp = cStringIO.StringIO()
else:
# Ensure the parent directories exist.
tinymce_zip_filename.dirname().makedirs_p()
tinymce_zip_fp = open(tinymce_zip_filename, mode='w')
try:
write_zip_file(tinymce_zip_fp, tinymce_zip_files(tinymce_dir))
except Exception, exc:
tinymce_zip_fp.close()
tinymce_zip_filename.remove()
raise tasks.BuildFailure(
'Error occurred creating tinymce.zip: %r' % (exc,))
finally:
if not tinymce_zip_fp.closed:
tinymce_zip_fp.close()
@easy.task
@easy.cmdopts([
('docs-output=', '', 'Output directory for documentation'),
('docs-config=', '', 'Configuration file for documentation'),
])
def build_docs(options):
"""Builds documentation for the project."""
# Epydoc smartly makes its own output directory if the output
# directory doesn't exist, but is not so smart that it will
# recursively create the parents of the output directory if they
# don't exist. Since we have the output directory path here we
# might as well create it rather than parsing out just the parents
# and leaving the directory itself to epydoc.
if not os.path.exists(options.docs_output):
os.makedirs(options.docs_output)
# NOTE(nathaniel): Epydoc actually imports modules during analysis,
# Melange's modules in turn import App Engine modules, and App Engine
# modules complain if the right Django and App Engine settings aren't
# in place at import time. Consequently, we must mutate the current
# environment to be that of an App Engine test before we can build
# Melange's documentation.
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
os.environ['SERVER_SOFTWARE'] = 'build'
appengine_testbed = testbed.Testbed()
appengine_testbed.activate()
appengine_testbed.init_datastore_v3_stub()
# NOTE(nathaniel): Deriving the options to pass to (epydoc.)cli.main this
# way is horsehockey, but epydoc doesn't actually expose a proper API.
stored_actual_argv = sys.argv
sys.argv = [
'unused_fake_executuable',
'--config=%s' % options.docs_config,
'--output=%s' % options.docs_output,
]
epydoc_options, epydoc_names = cli.parse_arguments()
sys.argv = stored_actual_argv
# NOTE(nathaniel): As of 13 January 2014 this call emits two false
# positive "Bad argument - expected name or tuple" errors. See
# https://sourceforge.net/p/epydoc/bugs/363/ for progress.
cli.main(epydoc_options, epydoc_names)
@easy.task
def deep_overrides(options):
"""Copies files from the copy structure to the build directory."""
dirs = [options.app_folder / i for i in options.copy_dirs]
dont_copy_dirs = [options.app_folder / i for i in options.dont_copy_dirs]
for source_dir in dirs:
dest_dir = options.app_build / options.app_folder.relpathto(source_dir)
dest_dir.remove()
source_dir.copytree(dest_dir)
for remove_dir in dont_copy_dirs:
dest_dir = options.app_build / options.app_folder.relpathto(remove_dir)
dest_dir.rmtree()
@easy.task
def overrides(options):
"""Copies files from the overrides structure to the build directory."""
for override_path in options.overrides_dirs:
target = options.app_build / override_path
unroll_symlink(target)
for override_path in options.overrides_files:
target = options.overrides_folder / override_path
if not target.exists():
continue
if not target.isfile():
tasks.environment.info('target "%s" is not a file', target)
continue
to = options.app_build / override_path
to.remove()
target.symlink(to)
def unroll_symlink(target):
"""Unrolls a symlink.
Does the following if target is a directory symlink:
- removes the symlink
- creates a directory with the same name
- populates it with symlinks to individual files
Otherwise does nothing.
"""
if not target.exists():
tasks.environment.info('target "%s" does not exist', target)
return
if not target.isdir():
tasks.environment.info('target "%s" is not a directory', target)
return
if not target.islink():
tasks.environment.info('target "%s" is not a symlink', target)
return
deref = target.readlinkabs()
target.remove()
target.mkdir()
contents = deref.listdir()
for symlink_path in contents:
symlink_path.symlink(target / symlink_path.name)