blob: 83ef676046221a347be53c4bc93bf7d9c80eb709 [file] [log] [blame]
# Copyright 2015 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Map-reduce pipelines that back up acceptance and rejection of proposals."""
from melange.appengine import django_setup
django_setup.setup_environment()
import logging
from google.appengine.ext import ndb
from mapreduce import base_handler
from mapreduce import context as mr_context
from mapreduce import mapreduce_pipeline
from melange.models import job_status as job_status_model
from summerofcode.logic import proposal as proposal_logic
from summerofcode.models import proposal as proposal_model
def findToBeAcceptedProposalsMapper(org_key):
"""Mapper that finds to-be-accepted proposals for the specified organization.
For each student with a proposal that is supposed to be accepted for the
organization, the mapper emits the following tuple for each proposal:
for the organization:
(serialized student key, serialized proposal key)
Additionally, for each student who has submitted at least one proposal to the
organization but does not have any to-be-accepted proposals, the mapper emits
the following tuple:
(serialized student key, '')
"""
org_key = ndb.Key.from_old_key(org_key)
organization = org_key.get()
context = mr_context.get()
params = context.mapreduce_spec.mapper.params
program_key = ndb.Key(urlsafe=params['program_key'])
if organization.program != program_key:
return
# it is assumed that no organization has more than 1000 proposals
all_proposals = proposal_model.Proposal.query(
proposal_model.Proposal.organization == org_key).fetch(1000)
# count proposals which have been accepted already
accepted_proposals = [
proposal for proposal in all_proposals
if proposal.status == proposal_model.Status.ACCEPTED]
remaining_slots = organization.slot_allocation - len(accepted_proposals)
# filter only the proposals which should be accepted
pending_proposals = [
proposal for proposal in all_proposals
if proposal.status == proposal_model.Status.PENDING]
pending_proposals.sort(
key=lambda proposal: proposal.total_score, reverse=True)
to_be_accepted_proposals = [
proposal for proposal in pending_proposals
if proposal.has_mentor and proposal.accept_as_project]
# proposals can be accepted only if the organization has remaining slots
to_be_accepted_proposals = to_be_accepted_proposals[:max(remaining_slots, 0)]
for proposal in to_be_accepted_proposals:
yield (proposal.key.parent().urlsafe(), proposal.key.urlsafe())
# for each student who submitted a proposal to this organization and
# the proposal is not going to be accepted by organization,
# the mapper emits the empty string.
# therefore, the reducer still processes this student.
all_student_keys = set(proposal.key.parent() for proposal in all_proposals)
accepted_student_keys = set(
proposal.key.parent() for proposal in to_be_accepted_proposals)
for student_key in all_student_keys - accepted_student_keys:
yield (student_key.urlsafe(), '')
def setToBeAcceptedProposalsReducer(student_key, proposal_keys):
"""Reducer that sets to-be-accepted proposals for the specified student based
on the data received from the mapper.
Args:
student_key (string): Serialized student key.
proposal_keys: (list<string>): List of serialized keys of proposals which
are to be accepted for the student. The list can also contain the empty
string. This values should be discarded, i.e. not stored in
to_be_accepted_proposals property.
"""
student_key = ndb.Key(urlsafe=student_key)
proposal_keys = [
ndb.Key(urlsafe=proposal_key)
for proposal_key in proposal_keys
if proposal_key != '']
@ndb.transactional
def txn():
student = student_key.get()
student.student_data.to_be_accepted_proposals = proposal_keys
student.put()
txn()
def acceptAndRejectProposalsReducer(student_key, proposal_keys):
"""Reducer that accepts and rejects proposals for the specified student based
on the data received from the mapper.
Args:
student_key (string): Serialized student key.
proposal_keys: (list<string>): List of serialized keys of proposals which
are to be accepted for the student. The list can also contain the empty
string. This values should be discarded, i.e. not stored in
to_be_accepted_proposals property.
"""
student_key = ndb.Key(urlsafe=student_key)
student = student_key.get()
to_be_accepted_proposal_keys = [
ndb.Key(urlsafe=proposal_key)
for proposal_key in proposal_keys
if proposal_key != '']
query = proposal_logic.queryProposalsForStudent(student_key)
all_proposals = query.fetch(1000)
to_be_rejected_proposal_keys = [
proposal.key for proposal in all_proposals
if (proposal.key not in to_be_accepted_proposal_keys and
proposal.status != proposal_model.Status.WITHDRAWN)]
for proposal_key in to_be_rejected_proposal_keys:
proposal_logic.rejectProposal(proposal_key)
if len(to_be_accepted_proposal_keys) == 1:
proposal_logic.acceptProposal(to_be_accepted_proposal_keys[0])
elif len(to_be_accepted_proposal_keys) > 1:
# accept the proposal which has the highest rank according to the student
for proposal_key in student.student_data.preference_of_proposals:
if proposal_key in to_be_accepted_proposal_keys:
proposal_logic.acceptProposal(proposal_key)
accepted_proposal_key = proposal_key
break
else:
logging.warning(
'None of the proposals that can be accepted (%s) for student %s are '
'present in their preferences (%s).',
','.join(to_be_accepted_proposal_keys), student.key,
','.join(student.student_data.preference_of_proposals))
# reject all other proposals
for proposal_key in to_be_accepted_proposal_keys:
if proposal_key != accepted_proposal_key:
proposal_logic.rejectProposal(proposal_key)
# TODO(daniel): send an email to program admins that a student who had
# a duplicate was accepted.
class FindDuplicatesPipeline(base_handler.PipelineBase):
"""Map-reduce pipeline that finds duplicates for programs."""
def run(self, program_key): # pylint: disable=arguments-differ
"""Runs the pipeline for the specified program.
Args:
program_key (str): Serialized version of the program key.
"""
yield mapreduce_pipeline.MapreducePipeline(
'FindDuplicates',
'summerofcode.mapreduce.proposal_acceptance'
'.findToBeAcceptedProposalsMapper',
'summerofcode.mapreduce.proposal_acceptance'
'.setToBeAcceptedProposalsReducer',
'mapreduce.input_readers.DatastoreKeyInputReader',
mapper_params={
'entity_kind': 'Organization',
'program_key': program_key})
class AcceptAndRejectProposalsPipeline(base_handler.PipelineBase):
"""Map-reduce pipeline that accepts and rejects proposals for programs."""
def run(self, program_key): # pylint: disable=arguments-differ
"""Runs the pipeline for the specified program.
Args:
program_key (str): Serialized version of the program key.
"""
yield mapreduce_pipeline.MapreducePipeline(
'AcceptAndRejectProposals',
'summerofcode.mapreduce.proposal_acceptance'
'.findToBeAcceptedProposalsMapper',
'summerofcode.mapreduce.proposal_acceptance'
'.acceptAndRejectProposalsReducer',
'mapreduce.input_readers.DatastoreKeyInputReader',
mapper_params={
'entity_kind': 'Organization',
'program_key': program_key})
# identifier of a job that sets proposals to be accepted for the program
# and finds students with duplicate proposals.
_SET_TO_BE_ACCEPTED_PROPOSALS_JOB_ID = 'set-to-be-accepted-proposals-%s'
@ndb.transactional
def updateSetToBeAcceptedProposalsJobStatus(program_key, **kwargs):
"""Updates job status for "to be accepted" job for the specified program.
Args:
program_key (db.Key): Key of a program.
kwargs: Properties to set for the job status.
"""
job_id = _SET_TO_BE_ACCEPTED_PROPOSALS_JOB_ID % program_key.name()
job_status = job_status_model.JobStatus(id=job_id, **kwargs)
job_status.put()
def getSetToBeAcceptedProposalsJobStatus(program_key):
"""Returns job status for "to be accepted proposals" job for the specified
program.
Args:
program_key (db.Key): Key of a program.
Returns (melange.models.job_status.JobStatus):
Job status entity or None if job status does not exist.
"""
job_id = _SET_TO_BE_ACCEPTED_PROPOSALS_JOB_ID % program_key.name()
return job_status_model.JobStatus.get_by_id(job_id)