blob: 80b5279f3b2ea363709fa13642ce073a651e496e [file] [log] [blame]
#!/usr/bin/env python
#
# Copyright 2009 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This module supplies an interactive shell with remote api configured.
Usage is simple:
App Engine interactive console
>>> from soc.models.program import Program
>>> gen = lambda: Program.all()
>>> it = deepFetch(gen)
>>> result = [i for i in it]
"""
import code
import getpass
import os
import sys
import logging
def auth_func():
"""Returns a tuple with username and password.
"""
if os.path.exists('.appengine_auth'):
try:
f = open('.appengine_auth')
name = f.readline()
pwd = f.readline()
# nip off the newlines
return name[:-1], pwd[:-1]
finally:
f.close()
logging.error("Could not read auth from .appengine_auth")
return raw_input('Username:'), getpass.getpass('Password:')
def deepFetch(queryGen, key=None, filters=None, batchSize=100):
"""Iterator that yields an entity in batches.
Args:
queryGen: should return a Query object
key: used to .filter() for __key__
filters: dict with additional filters
batchSize: how many entities to retrieve in one datastore call
Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook).
"""
from google.appengine.ext import db
# AppEngine will not fetch more than 1000 results
batchSize = min(batchSize, 1000)
query = None
done = False
count = 0
if key:
key = db.Key(key)
while not done:
print count # pylint: disable=print-statement
query = queryGen()
if filters:
for filter_key, value in filters.items():
query.filter(filter_key, value)
if key:
query.filter("__key__ > ", key)
results = query.fetch(batchSize)
for result in results:
count += 1
yield result
if batchSize > len(results):
done = True
else:
key = results[-1].key()
def deepFetchNDB(queryGen, batchSize=100):
"""Iterator that yields an entity in batches.
Args:
queryGen: should return a Query object
batchSize: how many entities to retrieve in one datastore call
"""
from google.appengine.datastore import datastore_query
from google.appengine.ext import db
from google.appengine.ext import ndb
# AppEngine will not fetch more than 1000 results
batchSize = min(batchSize, 1000)
query = None
done = False
count = 0
more = True
next_cursor = datastore_query.Cursor()
while next_cursor and more:
print count # pylint: disable=print-statement
query = queryGen()
results, next_cursor, more = query.fetch_page(
batchSize, start_cursor=next_cursor)
for result in results:
count += 1
yield result
def setupRemote(app_id, host=None):
"""Sets up execution for the specified remote."""
from google.appengine.ext.remote_api import remote_api_stub
if not host:
host = '%s.appspot.com' % app_id.split("~")[-1]
remote_api_stub.ConfigureRemoteDatastore(
app_id, '/_ah/remote_api', auth_func, host)
def setupOAuthRemote(app_id, host=None):
"""Sets up execution for the specified remote using OAuth auth.
"""
from google.appengine.ext.remote_api import remote_api_stub
from google.appengine.tools import appengine_rpc_httplib2
if not host:
host = '%s.appspot.com' % app_id.split("~")[-1]
scopes = ('https://www.googleapis.com/auth/appengine.apis',
'https://www.googleapis.com/auth/userinfo.email')
# Client ID and Client Secret for "native applications" as received
# in the Google Developers Console. The secret isn't actually secret.
CLIENT_ID = ('599661808252-touv3c8hqh1fqomn755fks5p328gmab7'
'.apps.googleusercontent.com')
CLIENT_NOTSOSECRET = 'T-PfHZshjN1HuDEEXzT2MZ0C'
get_user_credentials = (
appengine_rpc_httplib2.HttpRpcServerOAuth2.OAuth2Parameters(
access_token=None,
client_id=CLIENT_ID,
client_secret=CLIENT_NOTSOSECRET,
scope=scopes,
refresh_token=None,
credential_file='~/.melange_client_oauth2_tokens',
token_uri=None,
)
)
def _oauth2_rpc_server_factory(*args, **kwargs):
"""A wrapper around the HttpRpcServerOauth2 class factory.
Necessary so we can set auth_tries to > 1.
"""
return appengine_rpc_httplib2.HttpRpcServerOAuth2(
*args, auth_tries=2, **kwargs)
remote_api_stub.ConfigureRemoteApi(
app_id,
'/_ah/remote_api',
get_user_credentials,
servername=host,
save_cookies=True,
rpc_server_factory=_oauth2_rpc_server_factory)
remote_api_stub.MaybeInvokeAuthentication()
def remote(args, context=None):
"""Starts a shell with the datastore as remote_api_stub.
Args:
args: arguments from the user
context: locals that should be added to the shell
"""
if not context:
context = {}
app_id = args[0]
host = args[1] if len(args) > 1 else None
if app_id.startswith('dev~'):
setupRemote(app_id, host)
else:
setupOAuthRemote(app_id, host)
context['deepFetch'] = deepFetch
try:
from IPython.frontend.terminal.embed import TerminalInteractiveShell
shell = TerminalInteractiveShell(user_ns=context)
shell.mainloop()
except ImportError:
# IPython < 0.11
# Explicitly pass an empty list as arguments, because otherwise
# IPython would use sys.argv from this script.
try:
from IPython.Shell import IPShell
shell = IPShell(argv=[], user_ns=context)
shell.mainloop()
except ImportError:
# IPython not found, use the vanilla interpreter shell
code.interact(
'App Engine interactive console for %s' % (app_id,), None, context)
def setup():
"""Sets up the sys.path and environment for development."""
here = os.path.abspath(__file__)
here = os.path.join(os.path.dirname(here), '..')
here = os.path.normpath(here)
appengine_location = os.path.join(here, 'thirdparty', 'google_appengine')
extra_paths = [
here,
# gflags is required by google-api-python-client
os.path.join(appengine_location, 'lib', 'python-gflags'),
# oauth2client is part of google-api-python-client
os.path.join(appengine_location, 'lib', 'google-api-python-client'),
os.path.join(appengine_location, 'lib', 'django-1.5'),
os.path.join(appengine_location, 'lib', 'webob-1.2.3'),
os.path.join(appengine_location, 'lib', 'yaml', 'lib'),
os.path.join(appengine_location, 'lib', 'fancy_urllib'),
os.path.join(appengine_location, 'lib', 'simplejson'),
os.path.join(appengine_location, 'lib', 'protorpc-1.0'),
appengine_location,
os.path.join(here, 'build')]
sys.path = extra_paths + sys.path
os.environ['SERVER_SOFTWARE'] = 'Development Interactive Shell'
def setDjango():
"""Sets Django version used by the application."""
# TODO(daniel): this should be removed at some point as Interactive
# Shell does not use Django. This is currently required, because
# when main module is loaded, it imports a module which requires
# Django version to be set. This should be changed so that it is
# loaded only for Prod/Dev server.
# can't import this until after the setup() function has been called
# to update sys.path
from melange.appengine import django_setup
django_setup.setup_environment()
def main():
"""Convenience wrapper that calls setup and remote."""
if len(sys.argv) < 2:
# pylint: disable=print-statement
print "Usage: %s app_id [host]" % (sys.argv[0],)
sys.exit(1)
setup()
setDjango()
remote(sys.argv[1:])
import main as app_main
if __name__ == '__main__':
main()