1 /*
2  *  Make.org Core API
3  *  Copyright (C) 2018 Make.org
4  *
5  * This program is free software: you can redistribute it and/or modify
6  *  it under the terms of the GNU Affero General Public License as
7  *  published by the Free Software Foundation, either version 3 of the
8  *  License, or (at your option) any later version.
9  *
10  *  This program is distributed in the hope that it will be useful,
11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  *  GNU Affero General Public License for more details.
14  *
15  *  You should have received a copy of the GNU Affero General Public License
16  *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
17  *
18  */
19 
20 package org.make.api.technical.generator.fixtures
21 
22 import cats.implicits._
23 import akka.stream.scaladsl.{Sink, Source}
24 import cats.data.{NonEmptyList => Nel}
25 import grizzled.slf4j.Logging
26 import org.make.api.extensions.MakeSettingsComponent
27 import org.make.api.operation.{
28   CreateOperationOfQuestion,
29   OperationOfQuestionServiceComponent,
30   OperationServiceComponent
31 }
32 import org.make.api.organisation.OrganisationServiceComponent
33 import org.make.api.partner.PartnerServiceComponent
34 import org.make.api.proposal.{
35   ProposalServiceComponent,
36   RefuseProposalRequest,
37   UpdateQualificationRequest,
38   UpdateVoteRequest
39 }
40 import org.make.api.question.{PersistentQuestionServiceComponent, QuestionServiceComponent}
41 import org.make.api.sessionhistory.SessionHistoryCoordinatorServiceComponent
42 import org.make.api.tag.TagServiceComponent
43 import org.make.api.technical.ActorSystemComponent
44 import org.make.api.technical.ExecutorServiceHelper._
45 import org.make.api.technical.generator.EntitiesGen
46 import org.make.api.technical.security.SecurityConfigurationComponent
47 import org.make.api.user.UserServiceComponent
48 import org.make.core.operation.{OperationId, SimpleOperation}
49 import org.make.core.partner.Partner
50 import org.make.core.proposal.{ProposalId, ProposalStatus, VoteKey}
51 import org.make.core.question.{Question, QuestionId}
52 import org.make.core.reference.{Country, Language}
53 import org.make.core.session.SessionId
54 import org.make.core.tag.{Tag, TagId}
55 import org.make.core.user.{User, UserId}
56 import org.make.core.{DateHelper, RequestContext}
57 import org.scalacheck.Gen
58 import org.scalacheck.Gen.Parameters
59 import org.scalacheck.rng.Seed
60 
61 import java.util.concurrent.Executors
62 import scala.concurrent.{ExecutionContext, Future}
63 
64 trait FixturesService {
65   def generate(
66     maybeOperationId: Option[OperationId],
67     maybeQuestionId: Option[QuestionId],
68     maxProposals: Option[Int] = None,
69     forceCountries: Option[Nel[Country]] = None,
70     forceLanguage: Option[Language] = None
71   ): Future[FixtureResponse]
72 }
73 
74 trait FixturesServiceComponent {
75   def fixturesService: FixturesService
76 }
77 
78 trait DefaultFixturesServiceComponent extends FixturesServiceComponent with Logging {
79   this: OperationServiceComponent
80     with UserServiceComponent
81     with QuestionServiceComponent
82     with OperationOfQuestionServiceComponent
83     with ProposalServiceComponent
84     with TagServiceComponent
85     with OrganisationServiceComponent
86     with PartnerServiceComponent
87     with PersistentQuestionServiceComponent
88     with SecurityConfigurationComponent
89     with SessionHistoryCoordinatorServiceComponent
90     with ActorSystemComponent
91     with MakeSettingsComponent =>
92   override lazy val fixturesService: FixturesService = new DefaultFixturesService
93 
94   class DefaultFixturesService extends FixturesService {
95 
96     private val nThreads = 32
97     implicit private val executionContext: ExecutionContext =
98       Executors.newFixedThreadPool(nThreads).instrument("fixtures").toExecutionContext
99 
100     @SuppressWarnings(Array("org.wartremover.warts.OptionPartial"))
101     override def generate(
102       maybeOperationId: Option[OperationId],
103       maybeQuestionId: Option[QuestionId],
104       maxProposals: Option[Int] = None,
105       forceCountries: Option[Nel[Country]],
106       forceLanguage: Option[Language]
107     ): Future[FixtureResponse] = {
108       val futureAdmin: Future[User] = userService.getUserByEmail(makeSettings.defaultAdmin.email).flatMap {
109         case Some(user) => Future.successful(user)
110         case None       => Future.failed(new IllegalStateException())
111       }
112       for {
113         admin          <- futureAdmin
114         operationId    <- generateOperation(maybeOperationId, admin.userId)
115         questionId     <- generateQuestion(maybeQuestionId, operationId, forceCountries, forceLanguage)
116         users          <- generateUsers(questionId)
117         orgas          <- generateOrganisations
118         partners       <- generatePartners(questionId, orgas)
119         question       <- questionService.getQuestion(questionId).map(_.get)
120         tags           <- generateTags(question)
121         proposals      <- generateProposals(question, users ++ orgas, tags.map(_.tagId), admin.userId, maxProposals)
122         orgasVoteCount <- generateOrganisationVotes(orgas, proposals)
123       } yield FixtureResponse(
124         operationId = operationId,
125         questionId = questionId,
126         userCount = users.size,
127         organisationCount = orgas.size,
128         partnerCount = partners.size,
129         tagCount = tags.size,
130         proposalCount = proposals.size,
131         organisationsVoteCount = orgasVoteCount
132       )
133     }
134 
135     def generateOperation(maybeOperationId: Option[OperationId], adminUserId: UserId): Future[OperationId] = {
136       maybeOperationId match {
137         case Some(operationId) => Future.successful(operationId)
138         case None =>
139           val operation: SimpleOperation = EntitiesGen.genSimpleOperation.value
140           operationService.create(userId = adminUserId, slug = operation.slug, operationKind = operation.operationKind)
141       }
142     }
143 
144     def generateQuestion(
145       maybeQuestionId: Option[QuestionId],
146       operationId: OperationId,
147       forceCountries: Option[Nel[Country]],
148       forceLanguage: Option[Language]
149     ): Future[QuestionId] = {
150       maybeQuestionId match {
151         case Some(questionId) =>
152           questionService.getQuestion(questionId).flatMap {
153             case Some(question) =>
154               persistentQuestionService
155                 .modify(
156                   question.copy(
157                     countries = forceCountries.getOrElse(question.countries),
158                     languages = forceLanguage.fold(question.languages)(Nel.of(_))
159                   )
160                 )
161                 .map(_.questionId)
162             case None => Future.successful(questionId)
163           }
164         case None =>
165           val parameters: CreateOperationOfQuestion =
166             EntitiesGen.genCreateOperationOfQuestion(operationId, forceCountries, forceLanguage).value
167           operationOfQuestionService.create(parameters).map { question =>
168             logger.info(s"generated: question ${question.questionId}")
169             question.questionId
170           }
171       }
172     }
173 
174     def generateUsers(questionId: QuestionId): Future[Seq[User]] = {
175       val usersData = Gen.listOf(EntitiesGen.genUserRegisterData(Some(questionId))).value
176       logger.info(s"generating: ${usersData.size} users")
177       Source(usersData.distinctBy(_.email))
178         .mapAsync(16) { data =>
179           userService.register(data, RequestContext.empty)
180         }
181         .runWith(Sink.seq)
182     }
183 
184     def generateOrganisations: Future[Seq[User]] = {
185       val parameters = Parameters.default.withSize(20)
186       val orgasData = Gen.listOf(EntitiesGen.genOrganisationRegisterData).pureApply(parameters, Seed.random())
187       logger.info(s"generating: ${orgasData.size} organisations")
188       Source(orgasData.distinctBy(_.email))
189         .mapAsync(16) { data =>
190           organisationService.register(data, RequestContext.empty)
191         }
192         .runWith(Sink.seq)
193     }
194 
195     def generatePartners(questionId: QuestionId, organisations: Seq[User]): Future[Seq[Partner]] = {
196       val parameters = Parameters.default.withSize(20)
197       val partnersData =
198         Gen.listOf(EntitiesGen.genCreatePartnerRequest(None, questionId)).pureApply(parameters, Seed.random()) ++
199           organisations.map(orga => EntitiesGen.genCreatePartnerRequest(Some(orga), questionId).value)
200       logger.info(s"generating: ${partnersData.size} partners")
201       Source(partnersData)
202         .mapAsync(16) { data =>
203           partnerService.createPartner(data)
204         }
205         .runWith(Sink.seq)
206     }
207 
208     @SuppressWarnings(Array("org.wartremover.warts.OptionPartial"))
209     def generateProposals(
210       question: Question,
211       users: Seq[User],
212       tagsIds: Seq[TagId],
213       adminUserId: UserId,
214       proposalsSize: Option[Int]
215     ): Future[Seq[ProposalId]] = {
216       val parameters: Parameters = Parameters.default.withSize(2000)
217 
218       def gen[T]: Gen[T] => Gen[List[T]] = proposalsSize match {
219         case None       => Gen.listOf(_)
220         case Some(size) => Gen.listOfN(size, _)
221       }
222 
223       val proposalsData = gen(EntitiesGen.genProposal(question, users, tagsIds))
224         .pureApply(parameters, Seed.random())
225       logger.info(s"generating: ${proposalsData.size} proposals")
226 
227       Source(proposalsData)
228         .mapAsync(16) { proposal =>
229           proposalService
230             .propose(
231               user = users.find(_.userId == proposal.author).get,
232               requestContext = proposal.creationContext,
233               createdAt = proposal.createdAt.getOrElse(DateHelper.now()),
234               content = proposal.content,
235               question = question,
236               initialProposal = proposal.initialProposal,
237               submittedAsLanguage = proposal.creationContext.languageContext.language.getOrElse(Language("fr")),
238               isAnonymous = proposal.isAnonymous,
239               proposalType = proposal.proposalType
240             )
241             .map(proposalId => proposal.copy(proposalId = proposalId))
242         }
243         .mapAsync(16) { proposal =>
244           proposal.status match {
245             case ProposalStatus.Pending => Future.successful(proposal)
246             case ProposalStatus.Postponed =>
247               proposalService
248                 .postponeProposal(proposal.proposalId, adminUserId, RequestContext.empty)
249                 .map(_ => proposal)
250             case ProposalStatus.Accepted =>
251               proposalService
252                 .validateProposal(
253                   proposalId = proposal.proposalId,
254                   moderator = adminUserId,
255                   requestContext = RequestContext.empty,
256                   question = question,
257                   newContent = None,
258                   newContentTranslations = None,
259                   sendNotificationEmail = false,
260                   tags = proposal.tags
261                 )
262                 .map(_ => proposal)
263             case ProposalStatus.Refused =>
264               proposalService
265                 .refuseProposal(
266                   proposalId = proposal.proposalId,
267                   moderator = adminUserId,
268                   requestContext = RequestContext.empty,
269                   request = RefuseProposalRequest(sendNotificationEmail = false, refusalReason = proposal.refusalReason)
270                 )
271                 .map(_ => proposal)
272             case ProposalStatus.Archived => Future.successful(proposal)
273           }
274         }
275         .mapAsync(16) { proposal =>
276           val votes = proposal.votingOptions.map(
277             _.deprecatedVotesSeq
278               .map(
279                 v =>
280                   UpdateVoteRequest(
281                     key = v.key,
282                     count = Some(v.count),
283                     countVerified = Some(v.countVerified),
284                     countSequence = Some(v.countSequence),
285                     countSegment = Some(v.countSegment),
286                     qualifications = v.qualifications.map(
287                       q =>
288                         UpdateQualificationRequest(
289                           key = q.key,
290                           count = Some(q.count),
291                           countVerified = Some(q.countVerified),
292                           countSequence = Some(q.countSequence),
293                           countSegment = Some(q.countSegment)
294                         )
295                     )
296                   )
297               )
298           )
299           proposal.status match {
300             case ProposalStatus.Accepted =>
301               proposalService
302                 .updateVotes(
303                   proposalId = proposal.proposalId,
304                   moderator = adminUserId,
305                   requestContext = RequestContext.empty,
306                   updatedAt = proposal.updatedAt.getOrElse(DateHelper.now()),
307                   votes = votes.getOrElse(Seq.empty)
308                 )
309                 .as(proposal.proposalId)
310             case _ => Future.successful(proposal.proposalId)
311           }
312         }
313         .runWith(Sink.seq)
314     }
315 
316     def generateTags(question: Question): Future[Seq[Tag]] = {
317       val tagsData: Seq[Tag] = Gen.listOf(EntitiesGen.genTag(question.operationId, Some(question.questionId))).value
318       logger.info(s"generating: ${tagsData.size} tags")
319       Source(tagsData.distinctBy(_.label))
320         .mapAsync(16) { tag =>
321           tagService.createTag(
322             label = tag.label,
323             tagTypeId = tag.tagTypeId,
324             question = question,
325             display = tag.display,
326             weight = tag.weight
327           )
328         }
329         .runWith(Sink.seq)
330     }
331 
332     def generateOrganisationVotes(orgas: Seq[User], proposals: Seq[ProposalId]): Future[Int] = {
333       val sampleSize = if (proposals.sizeIs > 50) 50 else proposals.size
334       val proposalsSample = Gen.pick(sampleSize, proposals).value
335 
336       def proposalKey(proposalId: ProposalId, sessionId: SessionId): String =
337         proposalService.generateProposalKeyHash(
338           proposalId,
339           sessionId,
340           Some("fixtures"),
341           securityConfiguration.secureVoteSalt
342         )
343 
344       def requestContext(sessionId: SessionId) =
345         RequestContext.empty.copy(sessionId = sessionId, location = Some("fixtures"))
346 
347       Source(orgas)
348         .mapAsync(16) { orga =>
349           val sessionId = SessionId(orga.userId.value)
350           sessionHistoryCoordinatorService
351             .convertSession(sessionId, orga.userId, requestContext(sessionId))
352             .map(_ => orga)
353         }
354         .map(_ -> Gen.pick(sampleSize / 5, proposalsSample).value)
355         .mapAsync(16) {
356           case (orga, sample) =>
357             val sessionId = SessionId(orga.userId.value)
358             Source(sample.toSeq)
359               .mapAsync(1) { proposalId =>
360                 proposalService.voteProposal(
361                   proposalId = proposalId,
362                   maybeUserId = Some(orga.userId),
363                   requestContext = requestContext(sessionId),
364                   voteKey = Gen.frequency((6, VoteKey.Agree), (3, VoteKey.Disagree), (1, VoteKey.Neutral)).value,
365                   proposalKey = Some(proposalKey(proposalId, sessionId))
366                 )
367               }
368               .runWith(Sink.seq)
369               .map(_.size)
370         }
371         .runWith(Sink.seq)
372         .map(_.sum)
373     }
374   }
375 }