blob: 1e2caff46175a96fd6bf4de03530d83fd502b8a2 [file] [log] [blame]
# Copyright 2015 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Map-reduce scripts to update grading record entities."""
import logging
from melange.appengine import django_setup
django_setup.setup_environment()
from google.appengine.ext import db
from google.appengine.ext import ndb
from mapreduce import operation
# MapReduce requires import of processed model classes.
# pylint: disable=unused-import
from soc.modules.gsoc.models.grading_record import GSoCGradingRecord
from soc.modules.gsoc.models.project import GSoCProject
from summerofcode.models.project import Project
# pylint: enable=unused-import
def deleteDuplicateRecords(record_key):
"""Deletes old records which are duplicated in the datastore.
Args:
record_key: db.Key of GSoCGradingRecord entity.
"""
dry_run = True
project_key = record_key.parent()
if project_key.kind() == 'GSoCProject':
project = db.get(project_key)
if not project:
yield operation.counters.Increment('No project')
# It is likely that the key has form
# (User, user_id), (GSoCProfile, profile_id), (GSoCProject, project_id)
# The problem is that in the meantime GSoCProfile entities were
# updated so that they reflect that GSoCProfile was converted to Profile.
# Therefore, the structure is
# (User, user_id), (Profile, profile_id), (GSoCProject, project_id')
# We can find corresponding GSoCGradingRecord entities because all data
# was converted at that point.
# It is not trivial to find those entities to verify they exist but we
# can rely on the fact that each student had at most one project
old_profile_key = project_key.parent() # the old GSoCProfile key
new_profile_key = db.Key.from_path(
'Profile', old_profile_key.name(), parent=old_profile_key.parent())
projects = GSoCProject.all().ancestor(new_profile_key).fetch(1000)
if not projects:
logging.error('No projects for profile %s', new_profile_key)
yield operation.counters.Increment('No new project')
elif len(projects) > 1:
logging.error('More than one project for profile %s', new_profile_key)
yield operation.counters.Increment('More than one new project')
else:
project = projects[0]
old_grading_record = db.get(record_key)
survey_group_key = (
GSoCGradingRecord.grading_survey_group .get_value_for_datastore(
old_grading_record))
# find corresponding GSoCGradingRecords
grading_records = (
GSoCGradingRecord
.all()
.filter('grading_survey_group', survey_group_key)
.ancestor(project)
.fetch(1000))
if not grading_records:
logging.error('No grading record for %s', project.key())
yield operation.counters.Increment('No grading record')
elif len(grading_records) > 1:
logging.error('More than one grading records for %s', project.key())
yield operation.counters.Increment('More than one grading record')
else:
grading_record = grading_records[0]
# make sure that all properties are the same
property_names = GSoCGradingRecord.properties().keys()
wrong = False
for property_name in property_names:
if property_name == 'modified':
continue
old = getattr(old_grading_record, property_name)
new = getattr(grading_record, property_name)
# compare keys for reference properties
if property_name in (
'grading_survey_group', 'mentor_record', 'student_record'):
if old and not new:
logging.error('old != new, %s != %s', old.key(), new)
wrong = True
elif not old and new:
logging.error('old != new, %s != %s', old, new.key())
wrong = True
elif not old and not new:
continue
elif old.key() != new.key():
logging.error('old != new, %s != %s', old.key(), new.key())
wrong = True
# compare values for the rest of properties
elif old != new:
logging.error('old != new, %s != %s', old, new)
wrong = True
if wrong:
yield operation.counters.Increment('Different values')
else:
# when grading_record was created modified property was marked with
# auto_now=True. Now it is time to update it.
# auto_now is effective on every put operation so the model must
# be modified while this operation runs
# GSoCGradingRecord.modified.auto_now = False
grading_record.modified = old_grading_record.modified
if not dry_run:
grading_record.put()
# everything is OK. there is a newer entity with the same data
# we can get rid of the old entity
if not dry_run:
old_grading_record.delete()
yield operation.counters.Increment('Successfully processed')
def verifyRecordsHasProject(record_key):
"""Verifies that the specified record correspond to an existing project
Args:
record_key: db.Key of GSoCGradingRecord entity.
"""
project_key = record_key.parent()
if project_key.kind() == 'GSoCProject':
project = db.get(project_key)
if not project:
yield operation.counters.Increment('No DB project')
elif project_key.kind() == 'Project':
project = ndb.Key.from_old_key(project_key).get()
if not project:
yield operation.counters.Increment('No NDB project')
else:
yield operation.counters.Increment('Unknown kind: %s' % project_key.kind())
def updateProject(record_key):
"""Updates project corresponding to the specified record.
If the project has 'GSoCProject' kind, a new entity is created and its kind
is 'Project'.
Args:
record_key: db.Key of GSoCGradingRecord entity.
"""
dry_run = True
project_key = record_key.parent()
if project_key.kind() == 'GSoCProject':
yield operation.counters.Increment('GSoCProject kind')
project = db.get(project_key)
if not project.new_project:
logging.error('No new project for %s', project_key)
yield operation.counters.Increment('No new project')
else:
record = db.get(record_key)
new_project_key = db.Key(encoded=project.new_project)
properties = {
'grading_survey_group': (
GSoCGradingRecord
.grading_survey_group
.get_value_for_datastore(record)),
'mentor_record': (
GSoCGradingRecord
.mentor_record
.get_value_for_datastore(record)),
'student_record': (
GSoCGradingRecord
.student_record
.get_value_for_datastore(record)),
'grade_decision': record.grade_decision,
'locked': record.locked,
'created': record.created,
'modified': record.modified,
'parent': new_project_key,
}
new_record = GSoCGradingRecord(**properties)
if not dry_run:
new_record.put()
record.delete()
yield operation.counters.Increment('New record created')