1 /*
2  *  Make.org Core API
3  *  Copyright (C) 2018 Make.org
4  *
5  * This program is free software: you can redistribute it and/or modify
6  *  it under the terms of the GNU Affero General Public License as
7  *  published by the Free Software Foundation, either version 3 of the
8  *  License, or (at your option) any later version.
9  *
10  *  This program is distributed in the hope that it will be useful,
11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  *  GNU Affero General Public License for more details.
14  *
15  *  You should have received a copy of the GNU Affero General Public License
16  *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
17  *
18  */
19 
20 package org.make.api.proposal
21 
22 import enumeratum.{Enum, EnumEntry}
23 import grizzled.slf4j.Logging
24 import org.apache.commons.math3.distribution.BetaDistribution
25 import org.apache.commons.math3.random.{MersenneTwister, RandomGenerator}
26 import org.make.api.proposal.ProposalScorer.VotesCounter.{
27   SegmentVotesCounter,
28   SequenceVotesCounter,
29   VerifiedVotesVotesCounter
30 }
31 import org.make.api.proposal.ProposalScorer.Score
32 import org.make.core.proposal._
33 import org.make.core.proposal.VoteKey.{Agree, Disagree, Neutral}
34 import org.make.core.proposal.QualificationKey._
35 import org.make.core.proposal.Key._
36 import org.make.core.proposal.indexed.{SequencePool, Zone}
37 import org.make.core.sequence.SequenceConfiguration
38 import org.make.core.sequence.ExplorationSequenceConfiguration.SequenceThresholds
39 
40 final class ProposalScorer(val counts: VoteCounts, val verifiedCounts: VoteCounts, nonSequenceVotesWeight: Double) {
41 
42   private def tradeOff(generalScore: Double, specificScore: Double): Double =
43     nonSequenceVotesWeight * generalScore + (1 - nonSequenceVotesWeight) * specificScore
44 
45   private def individualScore(keyCount: Int, smoothing: Double, totalVotes: Int): Double =
46     Math.min((keyCount + smoothing) / (1 + totalVotes), 1)
47 
48   def score(keyCount: Int, keyVerifiedCount: Int, smoothing: Double): Double = {
49     tradeOff(
50       individualScore(keyVerifiedCount, smoothing, verifiedCounts.totalVotes),
51       individualScore(keyCount, smoothing, counts.totalVotes)
52     )
53   }
54 
55   private def confidence(keyCount: Int, smoothing: Double): Double = {
56     val specificVotesCount = counts.totalVotes
57     val specificVotes = individualScore(keyCount, smoothing, specificVotesCount)
58     ProposalScorer.confidenceInterval(specificVotes, specificVotesCount)
59   }
60 
61   private def keyScore[T <: Key: HasCount: HasSmoothing](key: T): Score = Score.forKey(this, key)
62 
63   lazy val agree: Score = keyScore(Agree)
64   lazy val disagree: Score = keyScore(Disagree)
65   lazy val neutral: Score = keyScore(Neutral)
66 
67   lazy val likeIt: Score = keyScore(LikeIt)
68   lazy val doable: Score = keyScore(Doable)
69   lazy val platitudeAgree: Score = keyScore(PlatitudeAgree)
70 
71   lazy val impossible: Score = keyScore(Impossible)
72   lazy val noWay: Score = keyScore(NoWay)
73   lazy val platitudeDisagree: Score = keyScore(PlatitudeDisagree)
74 
75   lazy val noOpinion: Score = keyScore(NoOpinion)
76   lazy val doNotCare: Score = keyScore(DoNotCare)
77   lazy val doNotUnderstand: Score = keyScore(DoNotUnderstand)
78 
79   lazy val platitude: Score = platitudeAgree + platitudeDisagree
80   lazy val topScore: Score = agree + likeIt + doable - noWay - impossible - platitude
81   lazy val controversy: Score = Score.min(agree, disagree) + Score.min(likeIt, noWay)
82   lazy val rejection: Score = disagree + noWay + impossible - likeIt - doable - platitude
83 
84   lazy val adhesion: Score = agree
85   lazy val greatness: Score = likeIt - noWay
86   lazy val realistic: Score = doable - impossible
87 
88   lazy val engagement: Score = greatness + realistic - platitude
89 
90   def zone(seqThreshold: SequenceThresholds): Zone =
91     zone(agree.score, disagree.score, neutral.score, seqThreshold)
92   def sampledZone(seqThreshold: SequenceThresholds): Zone =
93     zone(agree.sample, disagree.sample, neutral.sample, seqThreshold)
94 
95   /* Taken from the dial:
96   if (proposition['score_v2_adhesion'] >= .6) or \
97           (proposition['score_v2_neutral'] < .4 and proposition['score_v2_disagree'] < .15):
98       return 'consensus'
99   if (proposition['score_v2_disagree'] >= .6) or \
100           (proposition['score_v2_neutral'] < .4 and proposition['score_v2_adhesion'] < .15):
101       return 'rejection'
102   if proposition['score_v2_neutral'] >= .4:
103       return 'limbo'
104   return 'controversy'
105    */
106   private def zone(agree: Double, disagree: Double, neutral: Double, seqThreshold: SequenceThresholds): Zone = {
107     if (agree >= seqThreshold.strongOpinion.value || neutral < seqThreshold.neutralOpinion.value && disagree < seqThreshold.weakOpinion.value) {
108       Zone.Consensus
109     } else if (disagree >= seqThreshold.strongOpinion.value || (neutral < seqThreshold.neutralOpinion.value && agree < seqThreshold.weakOpinion.value)) {
110       Zone.Rejection
111     } else if (neutral >= seqThreshold.neutralOpinion.value) {
112       Zone.Limbo
113     } else {
114       Zone.Controversy
115     }
116   }
117 
118   def pool(configuration: SequenceConfiguration, status: ProposalStatus): SequencePool = {
119     val votesCount: Int = counts.totalVotes
120     val engagementRate: Double = engagement.lowerBound
121     val scoreRate: Double = topScore.lowerBound
122     val controversyRate: Double = controversy.lowerBound
123 
124     def isTestedFromScoreAndControversy: Boolean =
125       (configuration.testedProposalsScoreThreshold, configuration.testedProposalsControversyThreshold) match {
126         case (Some(scoreBase), Some(controversyBase)) => scoreRate >= scoreBase || controversyRate >= controversyBase
127         case (Some(scoreBase), _)                     => scoreRate >= scoreBase
128         case (_, Some(controversyBase))               => controversyRate >= controversyBase
129         case _                                        => true
130       }
131 
132     if (status == ProposalStatus.Accepted && votesCount < configuration.newProposalsVoteThreshold) {
133       SequencePool.New
134     } else if (status == ProposalStatus.Accepted &&
135                configuration.testedProposalsMaxVotesThreshold.forall(votesCount < _) &&
136                configuration.testedProposalsEngagementThreshold.forall(engagementRate > _) &&
137                isTestedFromScoreAndControversy) {
138       SequencePool.Tested
139     } else {
140       SequencePool.Excluded
141     }
142   }
143 }
144 
145 /**
146   * Define scores and the way to compute it.
147   *
148   * See https://gitlab.com/makeorg/data-science/make-data/-/blob/preproduction/make_data/score_v2.py
149   * it defines the computations from the dial.
150   */
151 object ProposalScorer extends Logging {
152 
153   def apply(votingOptions: VotingOptions, counter: VotesCounter, nonSequenceVotesWeight: Double): ProposalScorer = {
154     val counts = counter match {
155       case SequenceVotesCounter      => votingOptions.sequenceCounts
156       case SegmentVotesCounter       => votingOptions.segmentCounts
157       case VerifiedVotesVotesCounter => votingOptions.verifiedCounts
158     }
159     new ProposalScorer(counts, votingOptions.verifiedCounts, nonSequenceVotesWeight)
160   }
161 
162   private val random: RandomGenerator = new MersenneTwister()
163 
164   def setSeed(seed: Int): Unit = {
165     random.setSeed(seed)
166   }
167 
168   val votesSmoothing: Double = 0.33
169   val qualificationsSmoothing: Double = 0.01
170 
171   def confidenceInterval(probability: Double, observations: Double): Double = {
172     val smoothedObservations: Double = observations + 4
173     val smoothedProbability: Double = (observations * probability + 2) / smoothedObservations
174     val standardDeviation: Double = Math.sqrt(smoothedProbability * (1 - smoothedProbability) / smoothedObservations)
175     2 * standardDeviation
176   }
177 
178   final case class Score(score: Double, confidence: Double, sample: Double) {
179     val upperBound: Double = score + confidence
180     val lowerBound: Double = score - confidence
181 
182     def +(other: Score): Score = {
183       Score(
184         score = this.score + other.score,
185         confidence = sumConfidenceIntervals(this.confidence, other.confidence),
186         sample = this.sample + other.sample
187       )
188     }
189 
190     def -(other: Score): Score = {
191       Score(
192         score = this.score - other.score,
193         confidence = sumConfidenceIntervals(this.confidence, other.confidence),
194         sample = this.sample - other.sample
195       )
196     }
197   }
198 
199   object Score {
200     def min(first: Score, second: Score): Score = {
201       Score(
202         score = Math.min(first.score, second.score),
203         confidence = Math.max(first.confidence, second.confidence),
204         sample = Math.min(first.sample, second.sample)
205       )
206     }
207 
208     def forKey[T <: Key: HasCount: HasSmoothing](votes: ProposalScorer, key: T): Score = {
209       val successes = key.count(votes.counts)
210       val verifiedSuccesses = key.count(votes.verifiedCounts)
211       val trials = votes.counts.totalVotes
212       val smoothing = key.smoothing
213 
214       val sample = new BetaDistribution(random, successes + smoothing, trials - successes + 1).sample()
215       Score(
216         score = votes.score(successes, verifiedSuccesses, smoothing),
217         confidence = votes.confidence(successes, smoothing),
218         sample = sample
219       )
220     }
221   }
222 
223   private def sumConfidenceIntervals(firstConfidence: Double, secondConfidence: Double): Double =
224     Math.sqrt(Math.pow(firstConfidence, 2) + Math.pow(secondConfidence, 2))
225 
226   sealed abstract class VotesCounter extends EnumEntry
227 
228   object VotesCounter extends Enum[VotesCounter] {
229 
230     case object SequenceVotesCounter extends VotesCounter
231     case object SegmentVotesCounter extends VotesCounter
232     case object VerifiedVotesVotesCounter extends VotesCounter
233 
234     override val values: IndexedSeq[VotesCounter] = findValues
235   }
236 }