From b389c3d30201f466cc51a4fa397cc5e81c24fe50 Mon Sep 17 00:00:00 2001 From: twitter-team <> Date: Fri, 19 May 2023 09:53:56 -0700 Subject: [PATCH] Open-sourcing pushservice Pushservice is the main recommendation service we use to surface recommendations to our users via notifications. It fetches candidates from various sources, ranks them in order of relevance, and applies filters to determine the best one to send. --- README.md | 10 + pushservice/BUILD.bazel | 48 + pushservice/readme.md | 45 + .../main/python/models/heavy_ranking/BUILD | 169 + .../python/models/heavy_ranking/README.md | 20 + .../python/models/heavy_ranking/__init__.py | 0 .../python/models/heavy_ranking/deep_norm.py | 136 + .../main/python/models/heavy_ranking/eval.py | 59 + .../python/models/heavy_ranking/features.py | 138 + .../main/python/models/heavy_ranking/graph.py | 129 + .../python/models/heavy_ranking/lib/BUILD | 42 + .../python/models/heavy_ranking/lib/layers.py | 128 + .../python/models/heavy_ranking/lib/model.py | 76 + .../python/models/heavy_ranking/lib/params.py | 49 + .../models/heavy_ranking/model_pools.py | 34 + .../python/models/heavy_ranking/params.py | 89 + .../python/models/heavy_ranking/run_args.py | 59 + .../update_warm_start_checkpoint.py | 146 + pushservice/src/main/python/models/libs/BUILD | 16 + .../src/main/python/models/libs/__init__.py | 0 .../models/libs/customized_full_sparse.py | 56 + .../python/models/libs/get_feat_config.py | 176 + .../main/python/models/libs/graph_utils.py | 42 + .../main/python/models/libs/group_metrics.py | 114 + .../main/python/models/libs/initializer.py | 118 + .../models/libs/light_ranking_metrics.py | 255 + .../python/models/libs/metric_fn_utils.py | 294 + .../src/main/python/models/libs/model_args.py | 231 + .../main/python/models/libs/model_utils.py | 339 ++ .../python/models/libs/warm_start_utils.py | 309 + .../main/python/models/light_ranking/BUILD | 69 + .../python/models/light_ranking/README.md | 14 + .../python/models/light_ranking/__init__.py | 0 .../python/models/light_ranking/deep_norm.py | 226 + .../python/models/light_ranking/eval_model.py | 89 + .../models/light_ranking/model_pools_mlp.py | 187 + .../twitter/frigate/pushservice/BUILD.bazel | 337 ++ .../PushMixerThriftServerWarmupHandler.scala | 93 + .../frigate/pushservice/PushServiceMain.scala | 193 + .../ContentRecommenderMixerAdaptor.scala | 323 ++ ...EarlyBirdFirstDegreeCandidateAdaptor.scala | 293 + .../ExploreVideoTweetCandidateAdaptor.scala | 120 + .../adaptor/FRSTweetCandidateAdaptor.scala | 272 + .../adaptor/GenericCandidateAdaptor.scala | 107 + .../adaptor/HighQualityTweetsAdaptor.scala | 280 + .../ListsToRecommendCandidateAdaptor.scala | 152 + ...oggedOutPushCandidateSourceGenerator.scala | 54 + .../OnboardingPushCandidateAdaptor.scala | 101 + .../PushCandidateSourceGenerator.scala | 162 + .../TopTweetImpressionsCandidateAdaptor.scala | 326 ++ .../adaptor/TopTweetsByGeoAdaptor.scala | 413 ++ .../adaptor/TrendsCandidatesAdaptor.scala | 215 + .../adaptor/TripGeoCandidatesAdaptor.scala | 188 + .../frigate/pushservice/config/Config.scala | 461 ++ .../pushservice/config/DeployConfig.scala | 2150 +++++++ .../config/ExperimentsWithStats.scala | 16 + .../pushservice/config/ProdConfig.scala | 230 + .../pushservice/config/StagingConfig.scala | 193 + .../mlconfig/DeepbirdV2ModelConfig.scala | 23 + .../controller/PushServiceController.scala | 114 + ...DisplayLocationNotSupportedException.scala | 12 + .../InvalidSportDomainException.scala | 12 + .../TweetNTabRequestHydratorException.scala | 7 + .../exception/UnsupportedCrtException.scala | 11 + .../UttEntityNotFoundException.scala | 12 + .../pushservice/ml/HealthFeatureGetter.scala | 220 + .../ml/HydrationContextBuilder.scala | 179 + .../pushservice/ml/PushMLModelScorer.scala | 188 + .../pushservice/model/DiscoverTwitter.scala | 89 + .../model/F1FirstdegreeTweet.scala | 60 + .../ListRecommendationPushCandidate.scala | 72 + ...MagicFanoutCreatorEventPushCandidate.scala | 136 + .../model/MagicFanoutEventPushCandidate.scala | 303 + .../model/MagicFanoutHydratedCandidate.scala | 147 + .../model/MagicFanoutNewsEvent.scala | 99 + ...agicFanoutProductLaunchPushCandidate.scala | 95 + .../MagicFanoutSportsPushCandidate.scala | 119 + .../OutOfNetworkTweetPushCandidate.scala | 68 + .../frigate/pushservice/model/PushTypes.scala | 61 + .../model/ScheduledSpaceSpeaker.scala | 85 + .../model/ScheduledSpaceSubscriber.scala | 86 + .../SubscribedSearchTweetPushCandidate.scala | 56 + .../TopTweetImpressionsPushCandidate.scala | 70 + .../model/TopicProofTweetPushCandidate.scala | 71 + .../model/TrendTweetPushCandidate.scala | 50 + .../model/TripTweetPushCandidate.scala | 60 + .../pushservice/model/TweetAction.scala | 26 + .../pushservice/model/TweetFavorite.scala | 53 + .../pushservice/model/TweetRetweet.scala | 51 + .../model/candidate/CopyInfo.scala | 33 + .../model/candidate/MLScores.scala | 307 + .../model/candidate/QualityScribing.scala | 104 + .../pushservice/model/candidate/Scriber.scala | 277 + .../ibis/CustomConfigurationMapForIbis.scala | 25 + .../DiscoverTwitterPushIbis2Hydrator.scala | 17 + .../F1FirstDegreeTweetIbis2Hydrator.scala | 24 + .../model/ibis/Ibis2Hydrator.scala | 127 + .../ibis/InlineActionIbis2Hydrator.scala | 12 + .../model/ibis/ListIbis2Hydrator.scala | 21 + ...MagicFanoutCreatorEventIbis2Hydrator.scala | 29 + .../MagicFanoutNewsEventIbis2Hydrator.scala | 103 + ...agicFanoutProductLaunchIbis2Hydrator.scala | 54 + .../MagicFanoutSportsEventIbis2Hydrator.scala | 89 + .../ibis/OutOfNetworkTweetIbis2Hydrator.scala | 90 + .../model/ibis/OverrideForIbis2Request.scala | 210 + .../model/ibis/PushOverrideInfo.scala | 246 + .../RankedSocialContextIbis2Hydrator.scala | 22 + .../ScheduledSpaceSpeakerIbis2Hydrator.scala | 34 + ...cheduledSpaceSubscriberIbis2Hydrator.scala | 29 + .../SubscribedSearchTweetIbis2Hydrator.scala | 33 + ...eetImpressionsCandidateIbis2Hydrator.scala | 21 + .../ibis/TopicProofTweetIbis2Hydrator.scala | 32 + .../model/ibis/TrendTweetIbis2Hydrator.scala | 16 + .../ibis/TweetCandidateIbis2Hydrator.scala | 166 + .../ibis/TweetFavoriteIbis2Hydrator.scala | 21 + .../ibis/TweetRetweetIbis2Hydrator.scala | 32 + .../model/ntab/CandidateNTabCopy.scala | 21 + .../DiscoverTwitterNtabRequestHydrator.scala | 58 + .../model/ntab/EventNTabRequestHydrator.scala | 21 + ...1FirstDegreeTweetNTabRequestHydrator.scala | 18 + .../ListCandidateNTabRequestHydrator.scala | 34 + ...anoutCreatorEventNtabRequestHydrator.scala | 110 + ...icFanoutNewsEventNTabRequestHydrator.scala | 16 + ...noutProductLaunchNtabRequestHydrator.scala | 97 + ...FanoutSportsEventNTabRequestHydrator.scala | 95 + .../pushservice/model/ntab/NTabRequest.scala | 10 + .../model/ntab/NTabRequestHydrator.scala | 64 + .../model/ntab/NTabSocialContext.scala | 46 + ...OutOfNetworkTweetNTabRequestHydrator.scala | 78 + .../ScheduledSpaceNTabRequestHydrator.scala | 106 + ...cribedSearchTweetNtabRequestHydrator.scala | 23 + ...pTweetImpressionsNTabRequestHydrator.scala | 37 + .../TopicProofTweetNtabRequestHydrator.scala | 60 + .../model/ntab/TrendTweetNtabHydrator.scala | 61 + .../TweetFavoriteNTabRequestHydrator.scala | 38 + .../model/ntab/TweetNTabRequestHydrator.scala | 55 + .../TweetRetweetNTabRequestHydrator.scala | 38 + .../module/DeployConfigModule.scala | 68 + .../pushservice/module/FilterModule.scala | 16 + .../pushservice/module/FlagModule.scala | 56 + ...LoggedOutPushTargetUserBuilderModule.scala | 27 + .../module/PushHandlerModule.scala | 78 + .../module/PushServiceDarkTrafficModule.scala | 33 + .../module/PushTargetUserBuilderModule.scala | 64 + .../module/ThriftWebFormsModule.scala | 9 + .../pushservice/params/DeciderKey.scala | 210 + .../pushservice/params/PushConstants.scala | 126 + .../pushservice/params/PushEnums.scala | 135 + .../params/PushFeatureSwitchParams.scala | 5043 +++++++++++++++++ .../params/PushFeatureSwitches.scala | 751 +++ .../params/PushMLModelParams.scala | 60 + .../pushservice/params/PushParams.scala | 534 ++ .../params/PushServiceTunableKeys.scala | 9 + .../pushservice/params/ShardParams.scala | 3 + ...ingEpsilonGreedyExplorationPredicate.scala | 58 + .../predicate/BqmlHealthModelPredicates.scala | 129 + .../BqmlQualityModelPredicates.scala | 141 + .../CaretFeedbackHistoryFilter.scala | 99 + .../predicate/CasLockPredicate.scala | 45 + .../predicate/CrtDeciderPredicate.scala | 25 + .../predicate/DiscoverTwitterPredicate.scala | 47 + .../predicate/FatiguePredicate.scala | 74 + .../predicate/HealthPredicates.scala | 740 +++ .../JointDauAndQualityModelPredicate.scala | 39 + .../predicate/ListPredicates.scala | 110 + .../LoggedOutPreRankingPredicates.scala | 37 + .../predicate/LoggedOutTargetPredicates.scala | 53 + .../MlModelsHoldbackExperimentPredicate.scala | 71 + .../predicate/OONSpreadControlPredicate.scala | 116 + ...NTweetNegativeFeedbackBasedPredicate.scala | 82 + ...OfNetworkCandidatesQualityPredicates.scala | 221 + .../predicate/PNegMultimodalPredicates.scala | 83 + .../PostRankingPredicateHelper.scala | 50 + .../predicate/PreRankingPredicates.scala | 158 + .../predicate/PredicatesForCandidate.scala | 874 +++ .../predicate/SGSPredicatesForCandidate.scala | 174 + .../predicate/ScarecrowPredicate.scala | 138 + .../predicate/SpacePredicate.scala | 153 + .../predicate/TargetEngagementPredicate.scala | 27 + ...TargetNtabCaretClickFatiguePredicate.scala | 91 + .../predicate/TargetPredicates.scala | 292 + .../TopTweetImpressionsPredicates.scala | 56 + .../TweetEngagementRatioPredicate.scala | 112 + .../predicate/TweetLanguagePredicate.scala | 109 + .../TweetWithheldContentPredicate.scala | 35 + .../event/EventPredicatesForCandidate.scala | 155 + .../MagicFanoutPredicatesForCandidate.scala | 525 ++ .../MagicFanoutPredicatesUtil.scala | 218 + .../magic_fanout/MagicFanoutSportsUtil.scala | 231 + ...rgetingPredicateWrappersForCandidate.scala | 133 + ...BasedNtabCaretClickFatiguePredicates.scala | 973 ++++ .../ContinuousFunction.scala | 148 + .../ntab_caret_fatigue/FeedbackModel.scala | 136 + ...MagicFanoutNtabCaretFatiguePredicate.scala | 28 + ...bCaretClickCandidateFatiguePredicate.scala | 87 + .../NtabCaretClickFatiguePredicate.scala | 47 + .../NtabCaretClickFatigueUtils.scala | 108 + .../RecTypeNtabCaretFatiguePredicate.scala | 87 + .../pushservice/predicate/package.scala | 44 + .../OpenOrNtabClickQualityPredicate.scala | 27 + .../QualityPredicateCommon.scala | 165 + .../QualityPredicateMap.scala | 21 + .../pushservice/rank/CRTBoostRanker.scala | 54 + .../pushservice/rank/CRTDownRanker.scala | 45 + .../pushservice/rank/LoggedOutRanker.scala | 45 + .../pushservice/rank/ModelBasedRanker.scala | 204 + .../pushservice/rank/PushserviceRanker.scala | 31 + .../pushservice/rank/RFPHLightRanker.scala | 139 + .../frigate/pushservice/rank/RFPHRanker.scala | 297 + .../rank/SubscriptionCreatorRanker.scala | 110 + .../LoggedOutRefreshForPushHandler.scala | 259 + .../PushCandidateHydrator.scala | 239 + .../refresh_handler/RFPHFeatureHydrator.scala | 69 + .../refresh_handler/RFPHPrerankFilter.scala | 104 + .../refresh_handler/RFPHRestrictStep.scala | 34 + .../refresh_handler/RFPHStatsRecorder.scala | 77 + .../RefreshForPushHandler.scala | 292 + .../RefreshForPushNotifier.scala | 128 + .../cross/BaseCopyFramework.scala | 79 + .../cross/CandidateCopyExpansion.scala | 56 + .../cross/CandidateCopyPair.scala | 11 + .../cross/CandidateToCopy.scala | 263 + .../refresh_handler/cross/CopyFilters.scala | 41 + .../cross/CopyPredicates.scala | 36 + .../scriber/MrRequestScribeHandler.scala | 388 ++ .../send_handler/SendHandler.scala | 250 + .../SendHandlerPushCandidateHydrator.scala | 184 + .../generator/CandidateGenerator.scala | 17 + ...FanoutCreatorEventCandidateGenerator.scala | 70 + ...gicFanoutNewsEventCandidateGenerator.scala | 57 + ...anoutProductLaunchCandidateGenerator.scala | 54 + ...cFanoutSportsEventCandidateGenerator.scala | 153 + .../generator/PushRequestToCandidate.scala | 49 + ...eduledSpaceSpeakerCandidateGenerator.scala | 55 + ...ledSpaceSubscriberCandidateGenerator.scala | 55 + .../pushservice/store/ContentMixerStore.scala | 17 + .../store/CopySelectionServiceStore.scala | 15 + .../pushservice/store/CrMixerTweetStore.scala | 58 + .../store/ExploreRankerStore.scala | 28 + .../store/FollowRecommendationsStore.scala | 46 + .../frigate/pushservice/store/IbisStore.scala | 190 + .../store/InterestDiscoveryStore.scala | 16 + .../store/LabeledPushRecsDecideredStore.scala | 156 + .../pushservice/store/LexServiceStore.scala | 26 + .../pushservice/store/NTabHistoryStore.scala | 45 + .../store/OCFPromptHistoryStore.scala | 73 + .../store/OnlineUserHistoryStore.scala | 81 + .../pushservice/store/OpenAppUserStore.scala | 13 + .../SocialGraphServiceProcessStore.scala | 21 + .../store/SoftUserFollowingStore.scala | 61 + .../store/TweetImpressionsStore.scala | 19 + .../store/TweetTranslationStore.scala | 211 + .../store/UttEntityHydrationStore.scala | 79 + .../pushservice/take/CandidateNotifier.scala | 160 + .../LoggedOutRefreshForPushNotifier.scala | 118 + .../pushservice/take/NotificationSender.scala | 95 + .../take/NotificationServiceSender.scala | 273 + .../take/SendHandlerNotifier.scala | 86 + .../CandidateValidator.scala | 83 + .../RFPHCandidateValidator.scala | 27 + .../SendHandlerPostCandidateValidator.scala | 26 + .../SendHandlerPreCandidateValidator.scala | 24 + .../channel_selection/ChannelCandidate.scala | 24 + .../channel_selection/ChannelSelector.scala | 15 + .../NtabOnlyChannelSelector.scala | 21 + .../take/history/EventBusWriter.scala | 37 + .../take/history/HistoryWriter.scala | 49 + .../take/predicates/BasicRFPHPredicates.scala | 7 + .../BasicSendHandlerPredicates.scala | 13 + .../predicates/BasicTweetPredicates.scala | 104 + .../BasicTweetPredicatesForRFPH.scala | 41 + .../OutOfNetworkTweetPredicates.scala | 16 + .../predicates/TakeCommonPredicates.scala | 36 + .../CandidatePredicatesMap.scala | 75 + .../SendHandlerCandidatePredicatesMap.scala | 78 + .../pushservice/take/sender/Ibis2Sender.scala | 185 + .../pushservice/take/sender/NtabSender.scala | 237 + .../pushservice/target/CustomFSFields.scala | 98 + .../LoggedOutPushTargetUserBuilder.scala | 182 + .../target/PushTargetUserBuilder.scala | 694 +++ .../target/RFPHTargetPredicateGenerator.scala | 37 + .../target/TargetAppPermissions.scala | 10 + .../target/TargetScoringDetails.scala | 121 + .../pushservice/util/AdaptorUtils.scala | 15 + .../pushservice/util/AdhocStatsUtil.scala | 104 + .../util/Candidate2FrigateNotification.scala | 119 + .../util/CandidateHydrationUtil.scala | 439 ++ .../pushservice/util/CandidateUtil.scala | 138 + .../frigate/pushservice/util/CopyUtil.scala | 448 ++ .../util/EmailLandingPageExperimentUtil.scala | 92 + .../pushservice/util/FunctionalUtil.scala | 12 + .../pushservice/util/IbisScribeTargets.scala | 55 + .../pushservice/util/InlineActionUtil.scala | 219 + .../util/MediaAnnotationsUtil.scala | 52 + .../util/MinDurationModifierCalculator.scala | 187 + .../pushservice/util/MrUserStateUtil.scala | 16 + .../util/NsfwPersonalizationUtil.scala | 126 + .../util/OverrideNotificationUtil.scala | 230 + .../pushservice/util/PushAdaptorUtil.scala | 151 + .../util/PushAppPermissionUtil.scala | 49 + .../pushservice/util/PushCapUtil.scala | 184 + .../pushservice/util/PushDeviceUtil.scala | 57 + .../pushservice/util/PushIbisUtil.scala | 36 + .../pushservice/util/PushToHomeUtil.scala | 24 + .../pushservice/util/RFPHTakeStepUtil.scala | 114 + .../pushservice/util/RelationshipUtil.scala | 66 + .../util/ResponseStatsTrackUtils.scala | 42 + .../util/SendHandlerPredicateUtil.scala | 129 + .../frigate/pushservice/util/TopicsUtil.scala | 340 ++ 309 files changed, 42796 insertions(+) create mode 100644 pushservice/BUILD.bazel create mode 100644 pushservice/readme.md create mode 100644 pushservice/src/main/python/models/heavy_ranking/BUILD create mode 100644 pushservice/src/main/python/models/heavy_ranking/README.md create mode 100644 pushservice/src/main/python/models/heavy_ranking/__init__.py create mode 100644 pushservice/src/main/python/models/heavy_ranking/deep_norm.py create mode 100644 pushservice/src/main/python/models/heavy_ranking/eval.py create mode 100644 pushservice/src/main/python/models/heavy_ranking/features.py create mode 100644 pushservice/src/main/python/models/heavy_ranking/graph.py create mode 100644 pushservice/src/main/python/models/heavy_ranking/lib/BUILD create mode 100644 pushservice/src/main/python/models/heavy_ranking/lib/layers.py create mode 100644 pushservice/src/main/python/models/heavy_ranking/lib/model.py create mode 100644 pushservice/src/main/python/models/heavy_ranking/lib/params.py create mode 100644 pushservice/src/main/python/models/heavy_ranking/model_pools.py create mode 100644 pushservice/src/main/python/models/heavy_ranking/params.py create mode 100644 pushservice/src/main/python/models/heavy_ranking/run_args.py create mode 100644 pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.py create mode 100644 pushservice/src/main/python/models/libs/BUILD create mode 100644 pushservice/src/main/python/models/libs/__init__.py create mode 100644 pushservice/src/main/python/models/libs/customized_full_sparse.py create mode 100644 pushservice/src/main/python/models/libs/get_feat_config.py create mode 100644 pushservice/src/main/python/models/libs/graph_utils.py create mode 100644 pushservice/src/main/python/models/libs/group_metrics.py create mode 100644 pushservice/src/main/python/models/libs/initializer.py create mode 100644 pushservice/src/main/python/models/libs/light_ranking_metrics.py create mode 100644 pushservice/src/main/python/models/libs/metric_fn_utils.py create mode 100644 pushservice/src/main/python/models/libs/model_args.py create mode 100644 pushservice/src/main/python/models/libs/model_utils.py create mode 100644 pushservice/src/main/python/models/libs/warm_start_utils.py create mode 100644 pushservice/src/main/python/models/light_ranking/BUILD create mode 100644 pushservice/src/main/python/models/light_ranking/README.md create mode 100644 pushservice/src/main/python/models/light_ranking/__init__.py create mode 100644 pushservice/src/main/python/models/light_ranking/deep_norm.py create mode 100644 pushservice/src/main/python/models/light_ranking/eval_model.py create mode 100644 pushservice/src/main/python/models/light_ranking/model_pools_mlp.py create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.bazel create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.scala create mode 100644 pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.scala diff --git a/README.md b/README.md index ebb1361864..5bff490189 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,16 @@ The core components of the For You Timeline included in this repository are list | | [visibility-filters](visibilitylib/README.md) | Responsible for filtering Twitter content to support legal compliance, improve product quality, increase user trust, protect revenue through the use of hard-filtering, visible product treatments, and coarse-grained downranking. | | | [timelineranker](timelineranker/README.md) | Legacy service which provides relevance-scored tweets from the Earlybird Search Index and UTEG service. | +### Recommended Notifications + +The core components that power Recommended Notifications included in this repository are listed below: + +| Type | Component | Description | +|------------|------------|------------| +| Service | [pushservice](pushservice/README.md) | Main recommendation service at Twitter used to surface recommendations to our users via notifications. +| Ranking | [pushservice-light-ranker](pushservice/src/main/python/models/light_ranking/README.md) | Light Ranker model used by pushservice to rank Tweets. Bridges candidate generation and heavy ranking by pre-selecting highly-relevant candidates from the initial huge candidate pool. | +| | [pushservice-heavy-ranker](pushservice/src/main/python/models/heavy_ranking/README.md) | Multi-task learning model to predict the probabilities that the target users will open and engage with the sent notifications. | + ## Build and test code We include Bazel BUILD files for most components, but not a top-level BUILD or WORKSPACE file. We plan to add a more complete build and test system in the future. diff --git a/pushservice/BUILD.bazel b/pushservice/BUILD.bazel new file mode 100644 index 0000000000..12efdb2e61 --- /dev/null +++ b/pushservice/BUILD.bazel @@ -0,0 +1,48 @@ +alias( + name = "frigate-pushservice", + target = ":frigate-pushservice_lib", +) + +target( + name = "frigate-pushservice_lib", + dependencies = [ + "frigate/frigate-pushservice-opensource/src/main/scala/com/twitter/frigate/pushservice", + ], +) + +jvm_binary( + name = "bin", + basename = "frigate-pushservice", + main = "com.twitter.frigate.pushservice.PushServiceMain", + runtime_platform = "java11", + tags = ["bazel-compatible"], + dependencies = [ + "3rdparty/jvm/ch/qos/logback:logback-classic", + "finatra/inject/inject-logback/src/main/scala", + "frigate/frigate-pushservice-opensource/src/main/scala/com/twitter/frigate/pushservice", + "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", + "twitter-server/logback-classic/src/main/scala", + ], + excludes = [ + exclude("com.twitter.translations", "translations-twitter"), + exclude("org.apache.hadoop", "hadoop-aws"), + exclude("org.tensorflow"), + scala_exclude("com.twitter", "ckoia-scala"), + ], +) + +jvm_app( + name = "bundle", + basename = "frigate-pushservice-package-dist", + archive = "zip", + binary = ":bin", + tags = ["bazel-compatible"], +) + +python3_library( + name = "mr_model_constants", + sources = [ + "config/deepbird/constants.py", + ], + tags = ["bazel-compatible"], +) diff --git a/pushservice/readme.md b/pushservice/readme.md new file mode 100644 index 0000000000..99c20fcba3 --- /dev/null +++ b/pushservice/readme.md @@ -0,0 +1,45 @@ +# Pushservice + +Pushservice is the main push recommendation service at Twitter used to generate recommendation-based notifications for users. It currently powers two functionalities: + +- RefreshForPushHandler: This handler determines whether to send a recommendation push to a user based on their ID. It generates the best push recommendation item and coordinates with downstream services to deliver it +- SendHandler: This handler determines and manage whether send the push to users based on the given target user details and the provided push recommendation item + +## Overview + +### RefreshForPushHandler + +RefreshForPushHandler follows these steps: + +- Building Target and checking eligibility + - Builds a target user object based on the given user ID + - Performs target-level filterings to determine if the target is eligible for a recommendation push +- Fetch Candidates + - Retrieves a list of potential candidates for the push by querying various candidate sources using the target +- Candidate Hydration + - Hydrates the candidate details with batch calls to different downstream services. +- Pre-rank Filtering, also called Light Filtering + - Filters the hydrated candidates with lightweight RPC calls. +- Rank + - Perform feature hydration for candidates and target user + - Performs light ranking on candidates + - Performs heavy ranking on candidates +- Take Step, also called Heavy Filtering + - Takes the top-ranked candidates one by one and applies heavy filtering until one candidate passes all filter steps +- Send + - Calls the appropriate downstream service to deliver the eligible candidate as a push and in-app notification to the target user + +### SendHandler + +SendHandler follows these steps: + +- Building Target + - Builds a target user object based on the given user ID +- Candidate Hydration + - Hydrates the candidate details with batch calls to different downstream services. +- Feature Hydration + - Perform feature hydration for candidates and target user +- Take Step, also called Heavy Filtering + - Perform filterings and validation checking for the given candidate +- Send + - Calls the appropriate downstream service to deliver the given candidate as a push and/or in-app notification to the target user \ No newline at end of file diff --git a/pushservice/src/main/python/models/heavy_ranking/BUILD b/pushservice/src/main/python/models/heavy_ranking/BUILD new file mode 100644 index 0000000000..2c25693a98 --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/BUILD @@ -0,0 +1,169 @@ +python37_binary( + name = "update_warm_start_checkpoint", + source = "update_warm_start_checkpoint.py", + tags = ["no-mypy"], + dependencies = [ + ":deep_norm_lib", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:update_warm_start_checkpoint", + ], +) + +python3_library( + name = "params_lib", + sources = ["params.py"], + tags = ["no-mypy"], + dependencies = [ + "3rdparty/python/pydantic:default", + "src/python/twitter/deepbird/projects/magic_recs/v11/lib:params_lib", + ], +) + +python3_library( + name = "features_lib", + sources = ["features.py"], + tags = ["no-mypy"], + dependencies = [ + ":params_lib", + "src/python/twitter/deepbird/projects/magic_recs/libs", + "twml:twml-nodeps", + ], +) + +python3_library( + name = "model_pools_lib", + sources = ["model_pools.py"], + tags = ["no-mypy"], + dependencies = [ + ":features_lib", + ":params_lib", + "src/python/twitter/deepbird/projects/magic_recs/v11/lib:model_lib", + ], +) + +python3_library( + name = "graph_lib", + sources = ["graph.py"], + tags = ["no-mypy"], + dependencies = [ + ":params_lib", + "src/python/twitter/deepbird/projects/magic_recs/libs", + ], +) + +python3_library( + name = "run_args_lib", + sources = ["run_args.py"], + tags = ["no-mypy"], + dependencies = [ + ":features_lib", + ":params_lib", + "twml:twml-nodeps", + ], +) + +python3_library( + name = "deep_norm_lib", + sources = ["deep_norm.py"], + tags = ["no-mypy"], + dependencies = [ + ":features_lib", + ":graph_lib", + ":model_pools_lib", + ":params_lib", + ":run_args_lib", + "src/python/twitter/deepbird/projects/magic_recs/libs", + "src/python/twitter/deepbird/util/data", + "twml:twml-nodeps", + ], +) + +python3_library( + name = "eval_lib", + sources = ["eval.py"], + tags = ["no-mypy"], + dependencies = [ + ":features_lib", + ":graph_lib", + ":model_pools_lib", + ":params_lib", + ":run_args_lib", + "src/python/twitter/deepbird/projects/magic_recs/libs", + "twml:twml-nodeps", + ], +) + +python37_binary( + name = "deep_norm", + source = "deep_norm.py", + dependencies = [ + ":deep_norm_lib", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:deep_norm", + "twml", + ], +) + +python37_binary( + name = "eval", + source = "eval.py", + dependencies = [ + ":eval_lib", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:eval", + "twml", + ], +) + +python3_library( + name = "mlwf_libs", + tags = ["no-mypy"], + dependencies = [ + ":deep_norm_lib", + "twml", + ], +) + +python37_binary( + name = "train_model", + source = "deep_norm.py", + dependencies = [ + ":deep_norm_lib", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:train_model", + ], +) + +python37_binary( + name = "train_model_local", + source = "deep_norm.py", + dependencies = [ + ":deep_norm_lib", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:train_model_local", + "twml", + ], +) + +python37_binary( + name = "eval_model_local", + source = "eval.py", + dependencies = [ + ":eval_lib", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:eval_model_local", + "twml", + ], +) + +python37_binary( + name = "eval_model", + source = "eval.py", + dependencies = [ + ":eval_lib", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:eval_model", + ], +) + +python37_binary( + name = "mlwf_model", + source = "deep_norm.py", + dependencies = [ + ":mlwf_libs", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/heavy_ranking:mlwf_model", + ], +) diff --git a/pushservice/src/main/python/models/heavy_ranking/README.md b/pushservice/src/main/python/models/heavy_ranking/README.md new file mode 100644 index 0000000000..75336a09ce --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/README.md @@ -0,0 +1,20 @@ +# Notification Heavy Ranker Model + +## Model Context +There are 4 major components of Twitter notifications recommendation system: 1) candidate generation 2) light ranking 3) heavy ranking & 4) quality control. This notification heavy ranker model is the core ranking model for the personalised notifications recommendation. It's a multi-task learning model to predict the probabilities that the target users will open and engage with the sent notifications. + + +## Directory Structure +- BUILD: this file defines python library dependencies +- deep_norm.py: this file contains how to set up continuous training, model evaluation and model exporting for the notification heavy ranker model +- eval.py: the main python entry file to set up the overall model evaluation pipeline +- features.py: this file contains importing feature list and support functions for feature engineering +- graph.py: this file defines how to build the tensorflow graph with specified model architecture, loss function and training configuration +- model_pools.py: this file defines the available model types for the heavy ranker +- params.py: this file defines hyper-parameters used in the notification heavy ranker +- run_args.py: this file defines command line parameters to run model training & evaluation +- update_warm_start_checkpoint.py: this file contains the support to modify checkpoints of the given saved heavy ranker model +- lib/BUILD: this file defines python library dependencies for tensorflow model architecture +- lib/layers.py: this file defines different type of convolution layers to be used in the heavy ranker model +- lib/model.py: this file defines the module containing ClemNet, the heavy ranker model type +- lib/params.py: this file defines parameters used in the heavy ranker model diff --git a/pushservice/src/main/python/models/heavy_ranking/__init__.py b/pushservice/src/main/python/models/heavy_ranking/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pushservice/src/main/python/models/heavy_ranking/deep_norm.py b/pushservice/src/main/python/models/heavy_ranking/deep_norm.py new file mode 100644 index 0000000000..7db281b4af --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/deep_norm.py @@ -0,0 +1,136 @@ +""" +Training job for the heavy ranker of the push notification service. +""" +from datetime import datetime +import json +import os + +import twml + +from ..libs.metric_fn_utils import flip_disliked_labels, get_metric_fn +from ..libs.model_utils import read_config +from ..libs.warm_start_utils import get_feature_list_for_heavy_ranking, warm_start_checkpoint +from .features import get_feature_config +from .model_pools import ALL_MODELS +from .params import load_graph_params +from .run_args import get_training_arg_parser + +import tensorflow.compat.v1 as tf +from tensorflow.compat.v1 import logging + + +def main() -> None: + args, _ = get_training_arg_parser().parse_known_args() + logging.info(f"Parsed args: {args}") + + params = load_graph_params(args) + logging.info(f"Loaded graph params: {params}") + + param_file = os.path.join(args.save_dir, "params.json") + logging.info(f"Saving graph params to: {param_file}") + with tf.io.gfile.GFile(param_file, mode="w") as file: + json.dump(params.json(), file, ensure_ascii=False, indent=4) + + logging.info(f"Get Feature Config: {args.feature_list}") + feature_list = read_config(args.feature_list).items() + feature_config = get_feature_config( + data_spec_path=args.data_spec, + params=params, + feature_list_provided=feature_list, + ) + feature_list_path = args.feature_list + + warm_start_from = args.warm_start_from + if args.warm_start_base_dir: + logging.info(f"Get warm started model from: {args.warm_start_base_dir}.") + + continuous_binary_feat_list_save_path = os.path.join( + args.warm_start_base_dir, "continuous_binary_feat_list.json" + ) + warm_start_folder = os.path.join(args.warm_start_base_dir, "best_checkpoint") + job_name = os.path.basename(args.save_dir) + ws_output_ckpt_folder = os.path.join(args.warm_start_base_dir, f"warm_start_for_{job_name}") + if tf.io.gfile.exists(ws_output_ckpt_folder): + tf.io.gfile.rmtree(ws_output_ckpt_folder) + + tf.io.gfile.mkdir(ws_output_ckpt_folder) + + warm_start_from = warm_start_checkpoint( + warm_start_folder, + continuous_binary_feat_list_save_path, + feature_list_path, + args.data_spec, + ws_output_ckpt_folder, + ) + logging.info(f"Created warm_start_from_ckpt {warm_start_from}.") + + logging.info("Build Trainer.") + metric_fn = get_metric_fn("OONC_Engagement" if len(params.tasks) == 2 else "OONC", False) + + trainer = twml.trainers.DataRecordTrainer( + name="magic_recs", + params=args, + build_graph_fn=lambda *args: ALL_MODELS[params.model.name](params=params)(*args), + save_dir=args.save_dir, + run_config=None, + feature_config=feature_config, + metric_fn=flip_disliked_labels(metric_fn), + warm_start_from=warm_start_from, + ) + + logging.info("Build train and eval input functions.") + train_input_fn = trainer.get_train_input_fn(shuffle=True) + eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) + + learn = trainer.learn + if args.distributed or args.num_workers is not None: + learn = trainer.train_and_evaluate + + if not args.directly_export_best: + logging.info("Starting training") + start = datetime.now() + learn( + early_stop_minimize=False, + early_stop_metric="pr_auc_unweighted_OONC", + early_stop_patience=args.early_stop_patience, + early_stop_tolerance=args.early_stop_tolerance, + eval_input_fn=eval_input_fn, + train_input_fn=train_input_fn, + ) + logging.info(f"Total training time: {datetime.now() - start}") + else: + logging.info("Directly exporting the model") + + if not args.export_dir: + args.export_dir = os.path.join(args.save_dir, "exported_models") + + logging.info(f"Exporting the model to {args.export_dir}.") + start = datetime.now() + twml.contrib.export.export_fn.export_all_models( + trainer=trainer, + export_dir=args.export_dir, + parse_fn=feature_config.get_parse_fn(), + serving_input_receiver_fn=feature_config.get_serving_input_receiver_fn(), + export_output_fn=twml.export_output_fns.batch_prediction_continuous_output_fn, + ) + + logging.info(f"Total model export time: {datetime.now() - start}") + logging.info(f"The MLP directory is: {args.save_dir}") + + continuous_binary_feat_list_save_path = os.path.join( + args.save_dir, "continuous_binary_feat_list.json" + ) + logging.info( + f"Saving the list of continuous and binary features to {continuous_binary_feat_list_save_path}." + ) + continuous_binary_feat_list = get_feature_list_for_heavy_ranking( + feature_list_path, args.data_spec + ) + twml.util.write_file( + continuous_binary_feat_list_save_path, continuous_binary_feat_list, encode="json" + ) + + +if __name__ == "__main__": + main() + logging.info("Done.") diff --git a/pushservice/src/main/python/models/heavy_ranking/eval.py b/pushservice/src/main/python/models/heavy_ranking/eval.py new file mode 100644 index 0000000000..7f74472fb7 --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/eval.py @@ -0,0 +1,59 @@ +""" +Evaluation job for the heavy ranker of the push notification service. +""" + +from datetime import datetime + +import twml + +from ..libs.metric_fn_utils import get_metric_fn +from ..libs.model_utils import read_config +from .features import get_feature_config +from .model_pools import ALL_MODELS +from .params import load_graph_params +from .run_args import get_eval_arg_parser + +from tensorflow.compat.v1 import logging + + +def main(): + args, _ = get_eval_arg_parser().parse_known_args() + logging.info(f"Parsed args: {args}") + + params = load_graph_params(args) + logging.info(f"Loaded graph params: {params}") + + logging.info(f"Get Feature Config: {args.feature_list}") + feature_list = read_config(args.feature_list).items() + feature_config = get_feature_config( + data_spec_path=args.data_spec, + params=params, + feature_list_provided=feature_list, + ) + + logging.info("Build DataRecordTrainer.") + metric_fn = get_metric_fn("OONC_Engagement" if len(params.tasks) == 2 else "OONC", False) + + trainer = twml.trainers.DataRecordTrainer( + name="magic_recs", + params=args, + build_graph_fn=lambda *args: ALL_MODELS[params.model.name](params=params)(*args), + save_dir=args.save_dir, + run_config=None, + feature_config=feature_config, + metric_fn=metric_fn, + ) + + logging.info("Run the evaluation.") + start = datetime.now() + trainer._estimator.evaluate( + input_fn=trainer.get_eval_input_fn(repeat=False, shuffle=False), + steps=None if (args.eval_steps is not None and args.eval_steps < 0) else args.eval_steps, + checkpoint_path=args.eval_checkpoint, + ) + logging.info(f"Evaluating time: {datetime.now() - start}.") + + +if __name__ == "__main__": + main() + logging.info("Job done.") diff --git a/pushservice/src/main/python/models/heavy_ranking/features.py b/pushservice/src/main/python/models/heavy_ranking/features.py new file mode 100644 index 0000000000..ce6a2686a6 --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/features.py @@ -0,0 +1,138 @@ +import os +from typing import Dict + +from twitter.deepbird.projects.magic_recs.libs.model_utils import filter_nans_and_infs +import twml +from twml.layers import full_sparse, sparse_max_norm + +from .params import FeaturesParams, GraphParams, SparseFeaturesParams + +import tensorflow as tf +from tensorflow import Tensor +import tensorflow.compat.v1 as tf1 + + +FEAT_CONFIG_DEFAULT_VAL = 0 +DEFAULT_FEATURE_LIST_PATH = "./feature_list_default.yaml" +FEATURE_LIST_DEFAULT_PATH = os.path.join( + os.path.dirname(os.path.realpath(__file__)), DEFAULT_FEATURE_LIST_PATH +) + + +def get_feature_config(data_spec_path=None, feature_list_provided=[], params: GraphParams = None): + + a_string_feat_list = [feat for feat, feat_type in feature_list_provided if feat_type != "S"] + + builder = twml.contrib.feature_config.FeatureConfigBuilder( + data_spec_path=data_spec_path, debug=False + ) + + builder = builder.extract_feature_group( + feature_regexes=a_string_feat_list, + group_name="continuous_features", + default_value=FEAT_CONFIG_DEFAULT_VAL, + type_filter=["CONTINUOUS"], + ) + + builder = builder.extract_feature_group( + feature_regexes=a_string_feat_list, + group_name="binary_features", + type_filter=["BINARY"], + ) + + if params.model.features.sparse_features: + builder = builder.extract_features_as_hashed_sparse( + feature_regexes=a_string_feat_list, + hash_space_size_bits=params.model.features.sparse_features.bits, + type_filter=["DISCRETE", "STRING", "SPARSE_BINARY"], + output_tensor_name="sparse_not_continuous", + ) + + builder = builder.extract_features_as_hashed_sparse( + feature_regexes=[feat for feat, feat_type in feature_list_provided if feat_type == "S"], + hash_space_size_bits=params.model.features.sparse_features.bits, + type_filter=["SPARSE_CONTINUOUS"], + output_tensor_name="sparse_continuous", + ) + + builder = builder.add_labels([task.label for task in params.tasks] + ["label.ntabDislike"]) + + if params.weight: + builder = builder.define_weight(params.weight) + + return builder.build() + + +def dense_features(features: Dict[str, Tensor], training: bool) -> Tensor: + """ + Performs feature transformations on the raw dense features (continuous and binary). + """ + with tf.name_scope("dense_features"): + x = filter_nans_and_infs(features["continuous_features"]) + + x = tf.sign(x) * tf.math.log(tf.abs(x) + 1) + x = tf1.layers.batch_normalization( + x, momentum=0.9999, training=training, renorm=training, axis=1 + ) + x = tf.clip_by_value(x, -5, 5) + + transformed_continous_features = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x) + + binary_features = filter_nans_and_infs(features["binary_features"]) + binary_features = tf.dtypes.cast(binary_features, tf.float32) + + output = tf.concat([transformed_continous_features, binary_features], axis=1) + + return output + + +def sparse_features( + features: Dict[str, Tensor], training: bool, params: SparseFeaturesParams +) -> Tensor: + """ + Performs feature transformations on the raw sparse features. + """ + + with tf.name_scope("sparse_features"): + with tf.name_scope("sparse_not_continuous"): + sparse_not_continuous = full_sparse( + inputs=features["sparse_not_continuous"], + output_size=params.embedding_size, + use_sparse_grads=training, + use_binary_values=False, + ) + + with tf.name_scope("sparse_continuous"): + shape_enforced_input = twml.util.limit_sparse_tensor_size( + sparse_tf=features["sparse_continuous"], input_size_bits=params.bits, mask_indices=False + ) + + normalized_continuous_sparse = sparse_max_norm( + inputs=shape_enforced_input, is_training=training + ) + + sparse_continuous = full_sparse( + inputs=normalized_continuous_sparse, + output_size=params.embedding_size, + use_sparse_grads=training, + use_binary_values=False, + ) + + output = tf.concat([sparse_not_continuous, sparse_continuous], axis=1) + + return output + + +def get_features(features: Dict[str, Tensor], training: bool, params: FeaturesParams) -> Tensor: + """ + Performs feature transformations on the dense and sparse features and combine the resulting + tensors into a single one. + """ + with tf.name_scope("features"): + x = dense_features(features, training) + tf1.logging.info(f"Dense features: {x.shape}") + + if params.sparse_features: + x = tf.concat([x, sparse_features(features, training, params.sparse_features)], axis=1) + + return x diff --git a/pushservice/src/main/python/models/heavy_ranking/graph.py b/pushservice/src/main/python/models/heavy_ranking/graph.py new file mode 100644 index 0000000000..4188736ac6 --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/graph.py @@ -0,0 +1,129 @@ +""" +Graph class defining methods to obtain key quantities such as: + * the logits + * the probabilities + * the final score + * the loss function + * the training operator +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Dict + +from twitter.deepbird.hparam import HParams +import twml + +from ..libs.model_utils import generate_disliked_mask +from .params import GraphParams + +import tensorflow as tf +import tensorflow.compat.v1 as tf1 + + +class Graph(ABC): + def __init__(self, params: GraphParams): + self.params = params + + @abstractmethod + def get_logits(self, features: Dict[str, tf.Tensor], mode: tf.estimator.ModeKeys) -> tf.Tensor: + pass + + def get_probabilities(self, logits: tf.Tensor) -> tf.Tensor: + return tf.math.cumprod(tf.nn.sigmoid(logits), axis=1, name="probabilities") + + def get_task_weights(self, labels: tf.Tensor) -> tf.Tensor: + oonc_label = tf.reshape(labels[:, 0], shape=(-1, 1)) + task_weights = tf.concat([tf.ones_like(oonc_label), oonc_label], axis=1) + + n_labels = len(self.params.tasks) + task_weights = tf.reshape(task_weights[:, 0:n_labels], shape=(-1, n_labels)) + + return task_weights + + def get_loss(self, labels: tf.Tensor, logits: tf.Tensor, **kwargs: Any) -> tf.Tensor: + with tf.name_scope("weights"): + disliked_mask = generate_disliked_mask(labels) + + labels = tf.reshape(labels[:, 0:2], shape=[-1, 2]) + + labels = labels * tf.cast(tf.logical_not(disliked_mask), dtype=labels.dtype) + + with tf.name_scope("task_weight"): + task_weights = self.get_task_weights(labels) + + with tf.name_scope("batch_size"): + batch_size = tf.cast(tf.shape(labels)[0], dtype=tf.float32, name="batch_size") + + weights = task_weights / batch_size + + with tf.name_scope("loss"): + loss = tf.reduce_sum( + tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits) * weights, + ) + + return loss + + def get_score(self, probabilities: tf.Tensor) -> tf.Tensor: + with tf.name_scope("score_weight"): + score_weights = tf.constant([task.score_weight for task in self.params.tasks]) + score_weights = score_weights / tf.reduce_sum(score_weights, axis=0) + + with tf.name_scope("score"): + score = tf.reshape(tf.reduce_sum(probabilities * score_weights, axis=1), shape=[-1, 1]) + + return score + + def get_train_op(self, loss: tf.Tensor, twml_params) -> Any: + with tf.name_scope("optimizer"): + learning_rate = twml_params.learning_rate + optimizer = tf1.train.GradientDescentOptimizer(learning_rate=learning_rate) + + update_ops = set(tf1.get_collection(tf1.GraphKeys.UPDATE_OPS)) + with tf.control_dependencies(update_ops): + train_op = twml.optimizers.optimize_loss( + loss=loss, + variables=tf1.trainable_variables(), + global_step=tf1.train.get_global_step(), + optimizer=optimizer, + learning_rate=None, + ) + + return train_op + + def __call__( + self, + features: Dict[str, tf.Tensor], + labels: tf.Tensor, + mode: tf.estimator.ModeKeys, + params: HParams, + config=None, + ) -> Dict[str, tf.Tensor]: + training = mode == tf.estimator.ModeKeys.TRAIN + logits = self.get_logits(features=features, training=training) + probabilities = self.get_probabilities(logits=logits) + score = None + loss = None + train_op = None + + if mode == tf.estimator.ModeKeys.PREDICT: + score = self.get_score(probabilities=probabilities) + output = {"loss": loss, "train_op": train_op, "prediction": score} + + elif mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL): + loss = self.get_loss(labels=labels, logits=logits) + + if mode == tf.estimator.ModeKeys.TRAIN: + train_op = self.get_train_op(loss=loss, twml_params=params) + + output = {"loss": loss, "train_op": train_op, "output": probabilities} + + else: + raise ValueError( + f""" + Invalid mode. Possible values are: {tf.estimator.ModeKeys.PREDICT}, {tf.estimator.ModeKeys.TRAIN}, and {tf.estimator.ModeKeys.EVAL} + . Passed: {mode} + """ + ) + + return output diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/BUILD b/pushservice/src/main/python/models/heavy_ranking/lib/BUILD new file mode 100644 index 0000000000..a0ed713c44 --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/lib/BUILD @@ -0,0 +1,42 @@ +python3_library( + name = "params_lib", + sources = [ + "params.py", + ], + tags = [ + "bazel-compatible", + "no-mypy", + ], + dependencies = [ + "3rdparty/python/pydantic:default", + ], +) + +python3_library( + name = "layers_lib", + sources = [ + "layers.py", + ], + tags = [ + "bazel-compatible", + "no-mypy", + ], + dependencies = [ + ], +) + +python3_library( + name = "model_lib", + sources = [ + "model.py", + ], + tags = [ + "bazel-compatible", + "no-mypy", + ], + dependencies = [ + ":layers_lib", + ":params_lib", + "3rdparty/python/absl-py:default", + ], +) diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/layers.py b/pushservice/src/main/python/models/heavy_ranking/lib/layers.py new file mode 100644 index 0000000000..33dd6f012b --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/lib/layers.py @@ -0,0 +1,128 @@ +""" +Different type of convolution layers to be used in the ClemNet. +""" +from typing import Any + +import tensorflow as tf + + +class KerasConv1D(tf.keras.layers.Layer): + """ + Basic Conv1D layer in a wrapper to be compatible with ClemNet. + """ + + def __init__( + self, + kernel_size: int, + filters: int, + strides: int, + padding: str, + use_bias: bool = True, + kernel_initializer: str = "glorot_uniform", + bias_initializer: str = "zeros", + **kwargs: Any, + ): + super(KerasConv1D, self).__init__(**kwargs) + self.kernel_size = kernel_size + self.filters = filters + self.use_bias = use_bias + self.kernel_initializer = kernel_initializer + self.bias_initializer = bias_initializer + self.strides = strides + self.padding = padding + + def build(self, input_shape: tf.TensorShape) -> None: + assert ( + len(input_shape) == 3 + ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." + + self.features = input_shape[1] + + self.w = tf.keras.layers.Conv1D( + kernel_size=self.kernel_size, + filters=self.filters, + strides=self.strides, + padding=self.padding, + use_bias=self.use_bias, + kernel_initializer=self.kernel_initializer, + bias_initializer=self.bias_initializer, + name=self.name, + ) + + def call(self, inputs: tf.Tensor, **kwargs: Any) -> tf.Tensor: + return self.w(inputs) + + +class ChannelWiseDense(tf.keras.layers.Layer): + """ + Dense layer is applied to each channel separately. This is more memory and computationally + efficient than flattening the channels and performing single dense layers over it which is the + default behavior in tf1. + """ + + def __init__( + self, + output_size: int, + use_bias: bool, + kernel_initializer: str = "uniform_glorot", + bias_initializer: str = "zeros", + **kwargs: Any, + ): + super(ChannelWiseDense, self).__init__(**kwargs) + self.output_size = output_size + self.use_bias = use_bias + self.kernel_initializer = kernel_initializer + self.bias_initializer = bias_initializer + + def build(self, input_shape: tf.TensorShape) -> None: + assert ( + len(input_shape) == 3 + ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." + + input_size = input_shape[1] + channels = input_shape[2] + + self.kernel = self.add_weight( + name="kernel", + shape=(channels, input_size, self.output_size), + initializer=self.kernel_initializer, + trainable=True, + ) + + self.bias = self.add_weight( + name="bias", + shape=(channels, self.output_size), + initializer=self.bias_initializer, + trainable=self.use_bias, + ) + + def call(self, inputs: tf.Tensor, **kwargs: Any) -> tf.Tensor: + x = inputs + + transposed_x = tf.transpose(x, perm=[2, 0, 1]) + transposed_residual = ( + tf.transpose(tf.matmul(transposed_x, self.kernel), perm=[1, 0, 2]) + self.bias + ) + output = tf.transpose(transposed_residual, perm=[0, 2, 1]) + + return output + + +class ResidualLayer(tf.keras.layers.Layer): + """ + Layer implementing a 3D-residual connection. + """ + + def build(self, input_shape: tf.TensorShape) -> None: + assert ( + len(input_shape) == 3 + ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." + + def call(self, inputs: tf.Tensor, residual: tf.Tensor, **kwargs: Any) -> tf.Tensor: + shortcut = tf.keras.layers.Conv1D( + filters=int(residual.shape[2]), strides=1, kernel_size=1, padding="SAME", use_bias=False + )(inputs) + + output = tf.add(shortcut, residual) + + return output diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/model.py b/pushservice/src/main/python/models/heavy_ranking/lib/model.py new file mode 100644 index 0000000000..c6c8b1c6ba --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/lib/model.py @@ -0,0 +1,76 @@ +""" +Module containing ClemNet. +""" +from typing import Any + +from .layers import ChannelWiseDense, KerasConv1D, ResidualLayer +from .params import BlockParams, ClemNetParams + +import tensorflow as tf +import tensorflow.compat.v1 as tf1 + + +class Block2(tf.keras.layers.Layer): + """ + Possible ClemNet block. Architecture is as follow: + Optional(DenseLayer + BN + Act) + Optional(ConvLayer + BN + Act) + Optional(Residual Layer) + + """ + + def __init__(self, params: BlockParams, **kwargs: Any): + super(Block2, self).__init__(**kwargs) + self.params = params + + def build(self, input_shape: tf.TensorShape) -> None: + assert ( + len(input_shape) == 3 + ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." + + def call(self, inputs: tf.Tensor, training: bool) -> tf.Tensor: + x = inputs + if self.params.dense: + x = ChannelWiseDense(**self.params.dense.dict())(inputs=x, training=training) + x = tf1.layers.batch_normalization(x, momentum=0.9999, training=training, axis=1) + x = tf.keras.layers.Activation(self.params.activation)(x) + + if self.params.conv: + x = KerasConv1D(**self.params.conv.dict())(inputs=x, training=training) + x = tf1.layers.batch_normalization(x, momentum=0.9999, training=training, axis=1) + x = tf.keras.layers.Activation(self.params.activation)(x) + + if self.params.residual: + x = ResidualLayer()(inputs=inputs, residual=x) + + return x + + +class ClemNet(tf.keras.layers.Layer): + """ + A residual network stacking residual blocks composed of dense layers and convolutions. + """ + + def __init__(self, params: ClemNetParams, **kwargs: Any): + super(ClemNet, self).__init__(**kwargs) + self.params = params + + def build(self, input_shape: tf.TensorShape) -> None: + assert len(input_shape) in ( + 2, + 3, + ), f"Tensor shape must be of length 3. Passed tensor of shape {input_shape}." + + def call(self, inputs: tf.Tensor, training: bool) -> tf.Tensor: + if len(inputs.shape) < 3: + inputs = tf.expand_dims(inputs, axis=-1) + + x = inputs + for block_params in self.params.blocks: + x = Block2(block_params)(inputs=x, training=training) + + x = tf.keras.layers.Flatten(name="flattened")(x) + if self.params.top: + x = tf.keras.layers.Dense(units=self.params.top.n_labels, name="logits")(x) + + return x diff --git a/pushservice/src/main/python/models/heavy_ranking/lib/params.py b/pushservice/src/main/python/models/heavy_ranking/lib/params.py new file mode 100644 index 0000000000..721d6ed95c --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/lib/params.py @@ -0,0 +1,49 @@ +""" +Parameters used in ClemNet. +""" +from typing import List, Optional + +from pydantic import BaseModel, Extra, Field, PositiveInt + + +# checkstyle: noqa + + +class ExtendedBaseModel(BaseModel): + class Config: + extra = Extra.forbid + + +class DenseParams(ExtendedBaseModel): + name: Optional[str] + bias_initializer: str = "zeros" + kernel_initializer: str = "glorot_uniform" + output_size: PositiveInt + use_bias: bool = Field(True) + + +class ConvParams(ExtendedBaseModel): + name: Optional[str] + bias_initializer: str = "zeros" + filters: PositiveInt + kernel_initializer: str = "glorot_uniform" + kernel_size: PositiveInt + padding: str = "SAME" + strides: PositiveInt = 1 + use_bias: bool = Field(True) + + +class BlockParams(ExtendedBaseModel): + activation: Optional[str] + conv: Optional[ConvParams] + dense: Optional[DenseParams] + residual: Optional[bool] + + +class TopLayerParams(ExtendedBaseModel): + n_labels: PositiveInt + + +class ClemNetParams(ExtendedBaseModel): + blocks: List[BlockParams] = [] + top: Optional[TopLayerParams] diff --git a/pushservice/src/main/python/models/heavy_ranking/model_pools.py b/pushservice/src/main/python/models/heavy_ranking/model_pools.py new file mode 100644 index 0000000000..de59ee1a67 --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/model_pools.py @@ -0,0 +1,34 @@ +""" +Candidate architectures for each task's. +""" + +from __future__ import annotations + +from typing import Dict + +from .features import get_features +from .graph import Graph +from .lib.model import ClemNet +from .params import ModelTypeEnum + +import tensorflow as tf + + +class MagicRecsClemNet(Graph): + def get_logits(self, features: Dict[str, tf.Tensor], training: bool) -> tf.Tensor: + + with tf.name_scope("logits"): + inputs = get_features(features=features, training=training, params=self.params.model.features) + + with tf.name_scope("OONC_logits"): + model = ClemNet(params=self.params.model.architecture) + oonc_logit = model(inputs=inputs, training=training) + + with tf.name_scope("EngagementGivenOONC_logits"): + model = ClemNet(params=self.params.model.architecture) + eng_logits = model(inputs=inputs, training=training) + + return tf.concat([oonc_logit, eng_logits], axis=1) + + +ALL_MODELS = {ModelTypeEnum.clemnet: MagicRecsClemNet} diff --git a/pushservice/src/main/python/models/heavy_ranking/params.py b/pushservice/src/main/python/models/heavy_ranking/params.py new file mode 100644 index 0000000000..64a7de2b18 --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/params.py @@ -0,0 +1,89 @@ +import enum +import json +from typing import List, Optional + +from .lib.params import BlockParams, ClemNetParams, ConvParams, DenseParams, TopLayerParams + +from pydantic import BaseModel, Extra, NonNegativeFloat +import tensorflow.compat.v1 as tf + + +# checkstyle: noqa + + +class ExtendedBaseModel(BaseModel): + class Config: + extra = Extra.forbid + + +class SparseFeaturesParams(ExtendedBaseModel): + bits: int + embedding_size: int + + +class FeaturesParams(ExtendedBaseModel): + sparse_features: Optional[SparseFeaturesParams] + + +class ModelTypeEnum(str, enum.Enum): + clemnet: str = "clemnet" + + +class ModelParams(ExtendedBaseModel): + name: ModelTypeEnum + features: FeaturesParams + architecture: ClemNetParams + + +class TaskNameEnum(str, enum.Enum): + oonc: str = "OONC" + engagement: str = "Engagement" + + +class Task(ExtendedBaseModel): + name: TaskNameEnum + label: str + score_weight: NonNegativeFloat + + +DEFAULT_TASKS = [ + Task(name=TaskNameEnum.oonc, label="label", score_weight=0.9), + Task(name=TaskNameEnum.engagement, label="label.engagement", score_weight=0.1), +] + + +class GraphParams(ExtendedBaseModel): + tasks: List[Task] = DEFAULT_TASKS + model: ModelParams + weight: Optional[str] + + +DEFAULT_ARCHITECTURE_PARAMS = ClemNetParams( + blocks=[ + BlockParams( + activation="relu", + conv=ConvParams(kernel_size=3, filters=5), + dense=DenseParams(output_size=output_size), + residual=False, + ) + for output_size in [1024, 512, 256, 128] + ], + top=TopLayerParams(n_labels=1), +) + +DEFAULT_GRAPH_PARAMS = GraphParams( + model=ModelParams( + name=ModelTypeEnum.clemnet, + architecture=DEFAULT_ARCHITECTURE_PARAMS, + features=FeaturesParams(sparse_features=SparseFeaturesParams(bits=18, embedding_size=50)), + ), +) + + +def load_graph_params(args) -> GraphParams: + params = DEFAULT_GRAPH_PARAMS + if args.param_file: + with tf.io.gfile.GFile(args.param_file, mode="r+") as file: + params = GraphParams.parse_obj(json.load(file)) + + return params diff --git a/pushservice/src/main/python/models/heavy_ranking/run_args.py b/pushservice/src/main/python/models/heavy_ranking/run_args.py new file mode 100644 index 0000000000..1cc33a8e03 --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/run_args.py @@ -0,0 +1,59 @@ +from twml.trainers import DataRecordTrainer + +from .features import FEATURE_LIST_DEFAULT_PATH + + +def get_training_arg_parser(): + parser = DataRecordTrainer.add_parser_arguments() + + parser.add_argument( + "--feature_list", + default=FEATURE_LIST_DEFAULT_PATH, + type=str, + help="Which features to use for training", + ) + + parser.add_argument( + "--param_file", + default=None, + type=str, + help="Path to JSON file containing the graph parameters. If None, model will load default parameters.", + ) + + parser.add_argument( + "--directly_export_best", + default=False, + action="store_true", + help="whether to directly_export best_checkpoint", + ) + + parser.add_argument( + "--warm_start_from", default=None, type=str, help="model dir to warm start from" + ) + + parser.add_argument( + "--warm_start_base_dir", + default=None, + type=str, + help="latest ckpt in this folder will be used to ", + ) + + parser.add_argument( + "--model_type", + default=None, + type=str, + help="Which type of model to train.", + ) + return parser + + +def get_eval_arg_parser(): + parser = get_training_arg_parser() + parser.add_argument( + "--eval_checkpoint", + default=None, + type=str, + help="Which checkpoint to use for evaluation", + ) + + return parser diff --git a/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.py b/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.py new file mode 100644 index 0000000000..04887b9cfd --- /dev/null +++ b/pushservice/src/main/python/models/heavy_ranking/update_warm_start_checkpoint.py @@ -0,0 +1,146 @@ +""" +Model for modifying the checkpoints of the magic recs cnn Model with addition, deletion, and reordering +of continuous and binary features. +""" + +import os + +from twitter.deepbird.projects.magic_recs.libs.get_feat_config import FEATURE_LIST_DEFAULT_PATH +from twitter.deepbird.projects.magic_recs.libs.warm_start_utils_v11 import ( + get_feature_list_for_heavy_ranking, + mkdirp, + rename_dir, + rmdir, + warm_start_checkpoint, +) +import twml +from twml.trainers import DataRecordTrainer + +import tensorflow.compat.v1 as tf +from tensorflow.compat.v1 import logging + + +def get_arg_parser(): + parser = DataRecordTrainer.add_parser_arguments() + parser.add_argument( + "--model_type", + default="deepnorm_gbdt_inputdrop2_rescale", + type=str, + help="specify the model type to use.", + ) + + parser.add_argument( + "--model_trainer_name", + default="None", + type=str, + help="deprecated, added here just for api compatibility.", + ) + + parser.add_argument( + "--warm_start_base_dir", + default="none", + type=str, + help="latest ckpt in this folder will be used.", + ) + + parser.add_argument( + "--output_checkpoint_dir", + default="none", + type=str, + help="Output folder for warm started ckpt. If none, it will move warm_start_base_dir to backup, and overwrite it", + ) + + parser.add_argument( + "--feature_list", + default="none", + type=str, + help="Which features to use for training", + ) + + parser.add_argument( + "--old_feature_list", + default="none", + type=str, + help="Which features to use for training", + ) + + return parser + + +def get_params(args=None): + parser = get_arg_parser() + if args is None: + return parser.parse_args() + else: + return parser.parse_args(args) + + +def _main(): + opt = get_params() + logging.info("parse is: ") + logging.info(opt) + + if opt.feature_list == "none": + feature_list_path = FEATURE_LIST_DEFAULT_PATH + else: + feature_list_path = opt.feature_list + + if opt.warm_start_base_dir != "none" and tf.io.gfile.exists(opt.warm_start_base_dir): + if opt.output_checkpoint_dir == "none" or opt.output_checkpoint_dir == opt.warm_start_base_dir: + _warm_start_base_dir = os.path.normpath(opt.warm_start_base_dir) + "_backup_warm_start" + _output_folder_dir = opt.warm_start_base_dir + + rename_dir(opt.warm_start_base_dir, _warm_start_base_dir) + tf.logging.info(f"moved {opt.warm_start_base_dir} to {_warm_start_base_dir}") + else: + _warm_start_base_dir = opt.warm_start_base_dir + _output_folder_dir = opt.output_checkpoint_dir + + continuous_binary_feat_list_save_path = os.path.join( + _warm_start_base_dir, "continuous_binary_feat_list.json" + ) + + if opt.old_feature_list != "none": + tf.logging.info("getting old continuous_binary_feat_list") + continuous_binary_feat_list = get_feature_list_for_heavy_ranking( + opt.old_feature_list, opt.data_spec + ) + rmdir(continuous_binary_feat_list_save_path) + twml.util.write_file( + continuous_binary_feat_list_save_path, continuous_binary_feat_list, encode="json" + ) + tf.logging.info(f"Finish writting files to {continuous_binary_feat_list_save_path}") + + warm_start_folder = os.path.join(_warm_start_base_dir, "best_checkpoint") + if not tf.io.gfile.exists(warm_start_folder): + warm_start_folder = _warm_start_base_dir + + rmdir(_output_folder_dir) + mkdirp(_output_folder_dir) + + new_ckpt = warm_start_checkpoint( + warm_start_folder, + continuous_binary_feat_list_save_path, + feature_list_path, + opt.data_spec, + _output_folder_dir, + opt.model_type, + ) + logging.info(f"Created new ckpt {new_ckpt} from {warm_start_folder}") + + tf.logging.info("getting new continuous_binary_feat_list") + new_continuous_binary_feat_list_save_path = os.path.join( + _output_folder_dir, "continuous_binary_feat_list.json" + ) + continuous_binary_feat_list = get_feature_list_for_heavy_ranking( + feature_list_path, opt.data_spec + ) + rmdir(new_continuous_binary_feat_list_save_path) + twml.util.write_file( + new_continuous_binary_feat_list_save_path, continuous_binary_feat_list, encode="json" + ) + tf.logging.info(f"Finish writting files to {new_continuous_binary_feat_list_save_path}") + + +if __name__ == "__main__": + _main() diff --git a/pushservice/src/main/python/models/libs/BUILD b/pushservice/src/main/python/models/libs/BUILD new file mode 100644 index 0000000000..82a014ba55 --- /dev/null +++ b/pushservice/src/main/python/models/libs/BUILD @@ -0,0 +1,16 @@ +python3_library( + name = "libs", + sources = ["*.py"], + tags = [ + "bazel-compatible", + "no-mypy", + ], + dependencies = [ + "cortex/recsys/src/python/twitter/cortex/recsys/utils", + "magicpony/common/file_access/src/python/twitter/magicpony/common/file_access", + "src/python/twitter/cortex/ml/embeddings/deepbird", + "src/python/twitter/cortex/ml/embeddings/deepbird/grouped_metrics", + "src/python/twitter/deepbird/util/data", + "twml:twml-nodeps", + ], +) diff --git a/pushservice/src/main/python/models/libs/__init__.py b/pushservice/src/main/python/models/libs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pushservice/src/main/python/models/libs/customized_full_sparse.py b/pushservice/src/main/python/models/libs/customized_full_sparse.py new file mode 100644 index 0000000000..b41f7d6940 --- /dev/null +++ b/pushservice/src/main/python/models/libs/customized_full_sparse.py @@ -0,0 +1,56 @@ +# pylint: disable=no-member, arguments-differ, attribute-defined-outside-init, unused-argument +""" +Implementing Full Sparse Layer, allow specify use_binary_value in call() to +overide default action. +""" + +from twml.layers import FullSparse as defaultFullSparse +from twml.layers.full_sparse import sparse_dense_matmul + +import tensorflow.compat.v1 as tf + + +class FullSparse(defaultFullSparse): + def call(self, inputs, use_binary_values=None, **kwargs): # pylint: disable=unused-argument + """The logic of the layer lives here. + + Arguments: + inputs: + A SparseTensor or a list of SparseTensors. + If `inputs` is a list, all tensors must have same `dense_shape`. + + Returns: + - If `inputs` is `SparseTensor`, then returns `bias + inputs * dense_b`. + - If `inputs` is a `list[SparseTensor`, then returns + `bias + add_n([sp_a * dense_b for sp_a in inputs])`. + """ + + if use_binary_values is not None: + default_use_binary_values = use_binary_values + else: + default_use_binary_values = self.use_binary_values + + if isinstance(default_use_binary_values, (list, tuple)): + raise ValueError( + "use_binary_values can not be %s when inputs is %s" + % (type(default_use_binary_values), type(inputs)) + ) + + outputs = sparse_dense_matmul( + inputs, + self.weight, + self.use_sparse_grads, + default_use_binary_values, + name="sparse_mm", + partition_axis=self.partition_axis, + num_partitions=self.num_partitions, + compress_ids=self._use_compression, + cast_indices_dtype=self._cast_indices_dtype, + ) + + if self.bias is not None: + outputs = tf.nn.bias_add(outputs, self.bias) + + if self.activation is not None: + return self.activation(outputs) # pylint: disable=not-callable + return outputs diff --git a/pushservice/src/main/python/models/libs/get_feat_config.py b/pushservice/src/main/python/models/libs/get_feat_config.py new file mode 100644 index 0000000000..4d8b3e93c9 --- /dev/null +++ b/pushservice/src/main/python/models/libs/get_feat_config.py @@ -0,0 +1,176 @@ +import os + +from twitter.deepbird.projects.magic_recs.libs.metric_fn_utils import USER_AGE_FEATURE_NAME +from twitter.deepbird.projects.magic_recs.libs.model_utils import read_config +from twml.contrib import feature_config as contrib_feature_config + + +# checkstyle: noqa + +FEAT_CONFIG_DEFAULT_VAL = -1.23456789 + +DEFAULT_INPUT_SIZE_BITS = 18 + +DEFAULT_FEATURE_LIST_PATH = "./feature_list_default.yaml" +FEATURE_LIST_DEFAULT_PATH = os.path.join( + os.path.dirname(os.path.realpath(__file__)), DEFAULT_FEATURE_LIST_PATH +) + +DEFAULT_FEATURE_LIST_LIGHT_RANKING_PATH = "./feature_list_light_ranking.yaml" +FEATURE_LIST_DEFAULT_LIGHT_RANKING_PATH = os.path.join( + os.path.dirname(os.path.realpath(__file__)), DEFAULT_FEATURE_LIST_LIGHT_RANKING_PATH +) + +FEATURE_LIST_DEFAULT = read_config(FEATURE_LIST_DEFAULT_PATH).items() +FEATURE_LIST_LIGHT_RANKING_DEFAULT = read_config(FEATURE_LIST_DEFAULT_LIGHT_RANKING_PATH).items() + + +LABELS = ["label"] +LABELS_MTL = {"OONC": ["label"], "OONC_Engagement": ["label", "label.engagement"]} +LABELS_LR = { + "Sent": ["label.sent"], + "HeavyRankPosition": ["meta.ranking.is_top3"], + "HeavyRankProbability": ["meta.ranking.weighted_oonc_model_score"], +} + + +def _get_new_feature_config_base( + data_spec_path, + labels, + add_sparse_continous=True, + add_gbdt=True, + add_user_id=False, + add_timestamp=False, + add_user_age=False, + feature_list_provided=[], + opt=None, + run_light_ranking_group_metrics_in_bq=False, +): + """ + Getter of the feature config based on specification. + + Args: + data_spec_path: A string indicating the path of the data_spec.json file, which could be + either a local path or a hdfs path. + labels: A list of strings indicating the name of the label in the data spec. + add_sparse_continous: A bool indicating if sparse_continuous feature needs to be included. + add_gbdt: A bool indicating if gbdt feature needs to be included. + add_user_id: A bool indicating if user_id feature needs to be included. + add_timestamp: A bool indicating if timestamp feature needs to be included. This will be useful + for sequential models and meta learning models. + add_user_age: A bool indicating if the user age feature needs to be included. + feature_list_provided: A list of features thats need to be included. If not specified, will use + FEATURE_LIST_DEFAULT by default. + opt: A namespace of arguments indicating the hyparameters. + run_light_ranking_group_metrics_in_bq: A bool indicating if heavy ranker score info needs to be included to compute group metrics in BigQuery. + + Returns: + A twml feature config object. + """ + + input_size_bits = DEFAULT_INPUT_SIZE_BITS if opt is None else opt.input_size_bits + + feature_list = feature_list_provided if feature_list_provided != [] else FEATURE_LIST_DEFAULT + a_string_feat_list = [f[0] for f in feature_list if f[1] != "S"] + + builder = contrib_feature_config.FeatureConfigBuilder(data_spec_path=data_spec_path) + + builder = builder.extract_feature_group( + feature_regexes=a_string_feat_list, + group_name="continuous", + default_value=FEAT_CONFIG_DEFAULT_VAL, + type_filter=["CONTINUOUS"], + ) + + builder = builder.extract_features_as_hashed_sparse( + feature_regexes=a_string_feat_list, + output_tensor_name="sparse_no_continuous", + hash_space_size_bits=input_size_bits, + type_filter=["BINARY", "DISCRETE", "STRING", "SPARSE_BINARY"], + ) + + if add_gbdt: + builder = builder.extract_features_as_hashed_sparse( + feature_regexes=["ads\..*"], + output_tensor_name="gbdt_sparse", + hash_space_size_bits=input_size_bits, + ) + + if add_sparse_continous: + s_string_feat_list = [f[0] for f in feature_list if f[1] == "S"] + + builder = builder.extract_features_as_hashed_sparse( + feature_regexes=s_string_feat_list, + output_tensor_name="sparse_continuous", + hash_space_size_bits=input_size_bits, + type_filter=["SPARSE_CONTINUOUS"], + ) + + if add_user_id: + builder = builder.extract_feature("meta.user_id") + if add_timestamp: + builder = builder.extract_feature("meta.timestamp") + if add_user_age: + builder = builder.extract_feature(USER_AGE_FEATURE_NAME) + + if run_light_ranking_group_metrics_in_bq: + builder = builder.extract_feature("meta.trace_id") + builder = builder.extract_feature("meta.ranking.weighted_oonc_model_score") + + builder = builder.add_labels(labels).define_weight("meta.weight") + + return builder.build() + + +def get_feature_config_with_sparse_continuous( + data_spec_path, + feature_list_provided=[], + opt=None, + add_user_id=False, + add_timestamp=False, + add_user_age=False, +): + task_name = opt.task_name if getattr(opt, "task_name", None) is not None else "OONC" + if task_name not in LABELS_MTL: + raise ValueError("Invalid Task Name !") + + return _get_new_feature_config_base( + data_spec_path=data_spec_path, + labels=LABELS_MTL[task_name], + add_sparse_continous=True, + add_user_id=add_user_id, + add_timestamp=add_timestamp, + add_user_age=add_user_age, + feature_list_provided=feature_list_provided, + opt=opt, + ) + + +def get_feature_config_light_ranking( + data_spec_path, + feature_list_provided=[], + opt=None, + add_user_id=True, + add_timestamp=False, + add_user_age=False, + add_gbdt=False, + run_light_ranking_group_metrics_in_bq=False, +): + task_name = opt.task_name if getattr(opt, "task_name", None) is not None else "HeavyRankPosition" + if task_name not in LABELS_LR: + raise ValueError("Invalid Task Name !") + if not feature_list_provided: + feature_list_provided = FEATURE_LIST_LIGHT_RANKING_DEFAULT + + return _get_new_feature_config_base( + data_spec_path=data_spec_path, + labels=LABELS_LR[task_name], + add_sparse_continous=False, + add_gbdt=add_gbdt, + add_user_id=add_user_id, + add_timestamp=add_timestamp, + add_user_age=add_user_age, + feature_list_provided=feature_list_provided, + opt=opt, + run_light_ranking_group_metrics_in_bq=run_light_ranking_group_metrics_in_bq, + ) diff --git a/pushservice/src/main/python/models/libs/graph_utils.py b/pushservice/src/main/python/models/libs/graph_utils.py new file mode 100644 index 0000000000..4a4626a59c --- /dev/null +++ b/pushservice/src/main/python/models/libs/graph_utils.py @@ -0,0 +1,42 @@ +""" +Utilties that aid in building the magic recs graph. +""" + +import re + +import tensorflow.compat.v1 as tf + + +def get_trainable_variables(all_trainable_variables, trainable_regexes): + """Returns a subset of trainable variables for training. + + Given a collection of trainable variables, this will return all those that match the given regexes. + Will also log those variables. + + Args: + all_trainable_variables (a collection of trainable tf.Variable): The variables to search through. + trainable_regexes (a collection of regexes): Variables that match any regex will be included. + + Returns a list of tf.Variable + """ + if trainable_regexes is None or len(trainable_regexes) == 0: + tf.logging.info("No trainable regexes found. Not using get_trainable_variables behavior.") + return None + + assert any( + tf.is_tensor(var) for var in all_trainable_variables + ), f"Non TF variable found: {all_trainable_variables}" + trainable_variables = list( + filter( + lambda var: any(re.match(regex, var.name, re.IGNORECASE) for regex in trainable_regexes), + all_trainable_variables, + ) + ) + tf.logging.info(f"Using filtered trainable variables: {trainable_variables}") + + assert ( + trainable_variables + ), "Did not find trainable variables after filtering after filtering from {} number of vars originaly. All vars: {} and train regexes: {}".format( + len(all_trainable_variables), all_trainable_variables, trainable_regexes + ) + return trainable_variables diff --git a/pushservice/src/main/python/models/libs/group_metrics.py b/pushservice/src/main/python/models/libs/group_metrics.py new file mode 100644 index 0000000000..eeef3c5010 --- /dev/null +++ b/pushservice/src/main/python/models/libs/group_metrics.py @@ -0,0 +1,114 @@ +import os +import time + +from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.computation import ( + write_grouped_metrics_to_mldash, +) +from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.configuration import ( + ClassificationGroupedMetricsConfiguration, + NDCGGroupedMetricsConfiguration, +) +import twml + +from .light_ranking_metrics import ( + CGRGroupedMetricsConfiguration, + ExpectedLossGroupedMetricsConfiguration, + RecallGroupedMetricsConfiguration, +) + +import numpy as np +import tensorflow.compat.v1 as tf +from tensorflow.compat.v1 import logging + + +# checkstyle: noqa + + +def run_group_metrics(trainer, data_dir, model_path, parse_fn, group_feature_name="meta.user_id"): + + start_time = time.time() + logging.info("Evaluating with group metrics.") + + metrics = write_grouped_metrics_to_mldash( + trainer=trainer, + data_dir=data_dir, + model_path=model_path, + group_fn=lambda datarecord: str( + datarecord.discreteFeatures[twml.feature_id(group_feature_name)[0]] + ), + parse_fn=parse_fn, + metric_configurations=[ + ClassificationGroupedMetricsConfiguration(), + NDCGGroupedMetricsConfiguration(k=[5, 10, 20]), + ], + total_records_to_read=1000000000, + shuffle=False, + mldash_metrics_name="grouped_metrics", + ) + + end_time = time.time() + logging.info(f"Evaluated Group Metics: {metrics}.") + logging.info(f"Group metrics evaluation time {end_time - start_time}.") + + +def run_group_metrics_light_ranking( + trainer, data_dir, model_path, parse_fn, group_feature_name="meta.trace_id" +): + + start_time = time.time() + logging.info("Evaluating with group metrics.") + + metrics = write_grouped_metrics_to_mldash( + trainer=trainer, + data_dir=data_dir, + model_path=model_path, + group_fn=lambda datarecord: str( + datarecord.discreteFeatures[twml.feature_id(group_feature_name)[0]] + ), + parse_fn=parse_fn, + metric_configurations=[ + CGRGroupedMetricsConfiguration(lightNs=[50, 100, 200], heavyKs=[1, 3, 10, 20, 50]), + RecallGroupedMetricsConfiguration(n=[50, 100, 200], k=[1, 3, 10, 20, 50]), + ExpectedLossGroupedMetricsConfiguration(lightNs=[50, 100, 200]), + ], + total_records_to_read=10000000, + num_batches_to_load=50, + batch_size=1024, + shuffle=False, + mldash_metrics_name="grouped_metrics_for_light_ranking", + ) + + end_time = time.time() + logging.info(f"Evaluated Group Metics for Light Ranking: {metrics}.") + logging.info(f"Group metrics evaluation time {end_time - start_time}.") + + +def run_group_metrics_light_ranking_in_bq(trainer, params, checkpoint_path): + logging.info("getting Test Predictions for Light Ranking Group Metrics in BigQuery !!!") + eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) + info_pool = [] + + for result in trainer.estimator.predict( + eval_input_fn, checkpoint_path=checkpoint_path, yield_single_examples=False + ): + traceID = result["trace_id"] + pred = result["prediction"] + label = result["target"] + info = np.concatenate([traceID, pred, label], axis=1) + info_pool.append(info) + + info_pool = np.concatenate(info_pool) + + locname = "/tmp/000/" + if not os.path.exists(locname): + os.makedirs(locname) + + locfile = locname + params.pred_file_name + columns = ["trace_id", "model_prediction", "meta__ranking__weighted_oonc_model_score"] + np.savetxt(locfile, info_pool, delimiter=",", header=",".join(columns)) + tf.io.gfile.copy(locfile, params.pred_file_path + params.pred_file_name, overwrite=True) + + if os.path.isfile(locfile): + os.remove(locfile) + + logging.info("Done Prediction for Light Ranking Group Metrics in BigQuery.") diff --git a/pushservice/src/main/python/models/libs/initializer.py b/pushservice/src/main/python/models/libs/initializer.py new file mode 100644 index 0000000000..8bba00216b --- /dev/null +++ b/pushservice/src/main/python/models/libs/initializer.py @@ -0,0 +1,118 @@ +import numpy as np +from tensorflow.keras import backend as K + + +class VarianceScaling(object): + """Initializer capable of adapting its scale to the shape of weights. + With `distribution="normal"`, samples are drawn from a truncated normal + distribution centered on zero, with `stddev = sqrt(scale / n)` where n is: + - number of input units in the weight tensor, if mode = "fan_in" + - number of output units, if mode = "fan_out" + - average of the numbers of input and output units, if mode = "fan_avg" + With `distribution="uniform"`, + samples are drawn from a uniform distribution + within [-limit, limit], with `limit = sqrt(3 * scale / n)`. + # Arguments + scale: Scaling factor (positive float). + mode: One of "fan_in", "fan_out", "fan_avg". + distribution: Random distribution to use. One of "normal", "uniform". + seed: A Python integer. Used to seed the random generator. + # Raises + ValueError: In case of an invalid value for the "scale", mode" or + "distribution" arguments.""" + + def __init__( + self, + scale=1.0, + mode="fan_in", + distribution="normal", + seed=None, + fan_in=None, + fan_out=None, + ): + self.fan_in = fan_in + self.fan_out = fan_out + if scale <= 0.0: + raise ValueError("`scale` must be a positive float. Got:", scale) + mode = mode.lower() + if mode not in {"fan_in", "fan_out", "fan_avg"}: + raise ValueError( + "Invalid `mode` argument: " 'expected on of {"fan_in", "fan_out", "fan_avg"} ' "but got", + mode, + ) + distribution = distribution.lower() + if distribution not in {"normal", "uniform"}: + raise ValueError( + "Invalid `distribution` argument: " 'expected one of {"normal", "uniform"} ' "but got", + distribution, + ) + self.scale = scale + self.mode = mode + self.distribution = distribution + self.seed = seed + + def __call__(self, shape, dtype=None, partition_info=None): + fan_in = shape[-2] if self.fan_in is None else self.fan_in + fan_out = shape[-1] if self.fan_out is None else self.fan_out + + scale = self.scale + if self.mode == "fan_in": + scale /= max(1.0, fan_in) + elif self.mode == "fan_out": + scale /= max(1.0, fan_out) + else: + scale /= max(1.0, float(fan_in + fan_out) / 2) + if self.distribution == "normal": + stddev = np.sqrt(scale) / 0.87962566103423978 + return K.truncated_normal(shape, 0.0, stddev, dtype=dtype, seed=self.seed) + else: + limit = np.sqrt(3.0 * scale) + return K.random_uniform(shape, -limit, limit, dtype=dtype, seed=self.seed) + + def get_config(self): + return { + "scale": self.scale, + "mode": self.mode, + "distribution": self.distribution, + "seed": self.seed, + } + + +def customized_glorot_uniform(seed=None, fan_in=None, fan_out=None): + """Glorot uniform initializer, also called Xavier uniform initializer. + It draws samples from a uniform distribution within [-limit, limit] + where `limit` is `sqrt(6 / (fan_in + fan_out))` + where `fan_in` is the number of input units in the weight tensor + and `fan_out` is the number of output units in the weight tensor. + # Arguments + seed: A Python integer. Used to seed the random generator. + # Returns + An initializer.""" + return VarianceScaling( + scale=1.0, + mode="fan_avg", + distribution="uniform", + seed=seed, + fan_in=fan_in, + fan_out=fan_out, + ) + + +def customized_glorot_norm(seed=None, fan_in=None, fan_out=None): + """Glorot norm initializer, also called Xavier uniform initializer. + It draws samples from a uniform distribution within [-limit, limit] + where `limit` is `sqrt(6 / (fan_in + fan_out))` + where `fan_in` is the number of input units in the weight tensor + and `fan_out` is the number of output units in the weight tensor. + # Arguments + seed: A Python integer. Used to seed the random generator. + # Returns + An initializer.""" + return VarianceScaling( + scale=1.0, + mode="fan_avg", + distribution="normal", + seed=seed, + fan_in=fan_in, + fan_out=fan_out, + ) diff --git a/pushservice/src/main/python/models/libs/light_ranking_metrics.py b/pushservice/src/main/python/models/libs/light_ranking_metrics.py new file mode 100644 index 0000000000..b83fcf3ae7 --- /dev/null +++ b/pushservice/src/main/python/models/libs/light_ranking_metrics.py @@ -0,0 +1,255 @@ +from functools import partial + +from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.configuration import ( + GroupedMetricsConfiguration, +) +from twitter.cortex.ml.embeddings.deepbird.grouped_metrics.helpers import ( + extract_prediction_from_prediction_record, +) + + +# checkstyle: noqa + + +def score_loss_at_n(labels, predictions, lightN): + """ + Compute the absolute ScoreLoss ranking metric + Args: + labels (list) : A list of label values (HeavyRanking Reference) + predictions (list): A list of prediction values (LightRanking Predictions) + lightN (int): size of the list at which of Initial candidates to compute ScoreLoss. (LightRanking) + """ + assert len(labels) == len(predictions) + + if lightN <= 0: + return None + + labels_with_predictions = zip(labels, predictions) + labels_with_sorted_predictions = sorted( + labels_with_predictions, key=lambda x: x[1], reverse=True + )[:lightN] + labels_top1_light = max([label for label, _ in labels_with_sorted_predictions]) + labels_top1_heavy = max(labels) + + return labels_top1_heavy - labels_top1_light + + +def cgr_at_nk(labels, predictions, lightN, heavyK): + """ + Compute Cumulative Gain Ratio (CGR) ranking metric + Args: + labels (list) : A list of label values (HeavyRanking Reference) + predictions (list): A list of prediction values (LightRanking Predictions) + lightN (int): size of the list at which of Initial candidates to compute CGR. (LightRanking) + heavyK (int): size of the list at which of Refined candidates to compute CGR. (HeavyRanking) + """ + assert len(labels) == len(predictions) + + if (not lightN) or (not heavyK): + out = None + elif lightN <= 0 or heavyK <= 0: + out = None + else: + + labels_with_predictions = zip(labels, predictions) + labels_with_sorted_predictions = sorted( + labels_with_predictions, key=lambda x: x[1], reverse=True + )[:lightN] + labels_topN_light = [label for label, _ in labels_with_sorted_predictions] + + if lightN <= heavyK: + cg_light = sum(labels_topN_light) + else: + labels_topK_heavy_from_light = sorted(labels_topN_light, reverse=True)[:heavyK] + cg_light = sum(labels_topK_heavy_from_light) + + ideal_ordering = sorted(labels, reverse=True) + cg_heavy = sum(ideal_ordering[: min(lightN, heavyK)]) + + out = 0.0 + if cg_heavy != 0: + out = max(cg_light / cg_heavy, 0) + + return out + + +def _get_weight(w, atK): + if not w: + return 1.0 + elif len(w) <= atK: + return 0.0 + else: + return w[atK] + + +def recall_at_nk(labels, predictions, n=None, k=None, w=None): + """ + Recall at N-K ranking metric + Args: + labels (list): A list of label values + predictions (list): A list of prediction values + n (int): size of the list at which of predictions to compute recall. (Light Ranking Predictions) + The default is None in which case the length of the provided predictions is used as L + k (int): size of the list at which of labels to compute recall. (Heavy Ranking Predictions) + The default is None in which case the length of the provided labels is used as L + w (list): weight vector sorted by labels + """ + assert len(labels) == len(predictions) + + if not any(labels): + out = None + else: + + safe_n = len(predictions) if not n else min(len(predictions), n) + safe_k = len(labels) if not k else min(len(labels), k) + + labels_with_predictions = zip(labels, predictions) + sorted_labels_with_predictions = sorted( + labels_with_predictions, key=lambda x: x[0], reverse=True + ) + + order_sorted_labels_predictions = zip(range(len(labels)), *zip(*sorted_labels_with_predictions)) + + order_with_predictions = [ + (order, pred) for order, label, pred in order_sorted_labels_predictions + ] + order_with_sorted_predictions = sorted(order_with_predictions, key=lambda x: x[1], reverse=True) + + pred_sorted_order_at_n = [order for order, _ in order_with_sorted_predictions][:safe_n] + + intersection_weight = [ + _get_weight(w, order) if order < safe_k else 0 for order in pred_sorted_order_at_n + ] + + intersection_score = sum(intersection_weight) + full_score = sum(w) if w else float(safe_k) + + out = 0.0 + if full_score != 0: + out = intersection_score / full_score + + return out + + +class ExpectedLossGroupedMetricsConfiguration(GroupedMetricsConfiguration): + """ + This is the Expected Loss Grouped metric computation configuration. + """ + + def __init__(self, lightNs=[]): + """ + Args: + lightNs (list): size of the list at which of Initial candidates to compute Expected Loss. (LightRanking) + """ + self.lightNs = lightNs + + @property + def name(self): + return "ExpectedLoss" + + @property + def metrics_dict(self): + metrics_to_compute = {} + for lightN in self.lightNs: + metric_name = "ExpectedLoss_atLight_" + str(lightN) + metrics_to_compute[metric_name] = partial(score_loss_at_n, lightN=lightN) + return metrics_to_compute + + def extract_label(self, prec, drec, drec_label): + return drec_label + + def extract_prediction(self, prec, drec, drec_label): + return extract_prediction_from_prediction_record(prec) + + +class CGRGroupedMetricsConfiguration(GroupedMetricsConfiguration): + """ + This is the Cumulative Gain Ratio (CGR) Grouped metric computation configuration. + CGR at the max length of each session is the default. + CGR at additional positions can be computed by specifying a list of 'n's and 'k's + """ + + def __init__(self, lightNs=[], heavyKs=[]): + """ + Args: + lightNs (list): size of the list at which of Initial candidates to compute CGR. (LightRanking) + heavyK (int): size of the list at which of Refined candidates to compute CGR. (HeavyRanking) + """ + self.lightNs = lightNs + self.heavyKs = heavyKs + + @property + def name(self): + return "cgr" + + @property + def metrics_dict(self): + metrics_to_compute = {} + for lightN in self.lightNs: + for heavyK in self.heavyKs: + metric_name = "cgr_atLight_" + str(lightN) + "_atHeavy_" + str(heavyK) + metrics_to_compute[metric_name] = partial(cgr_at_nk, lightN=lightN, heavyK=heavyK) + return metrics_to_compute + + def extract_label(self, prec, drec, drec_label): + return drec_label + + def extract_prediction(self, prec, drec, drec_label): + return extract_prediction_from_prediction_record(prec) + + +class RecallGroupedMetricsConfiguration(GroupedMetricsConfiguration): + """ + This is the Recall Grouped metric computation configuration. + Recall at the max length of each session is the default. + Recall at additional positions can be computed by specifying a list of 'n's and 'k's + """ + + def __init__(self, n=[], k=[], w=[]): + """ + Args: + n (list): A list of ints. List of prediction rank thresholds (for light) + k (list): A list of ints. List of label rank thresholds (for heavy) + """ + self.predN = n + self.labelK = k + self.weight = w + + @property + def name(self): + return "group_recall" + + @property + def metrics_dict(self): + metrics_to_compute = {"group_recall_unweighted": recall_at_nk} + if not self.weight: + metrics_to_compute["group_recall_weighted"] = partial(recall_at_nk, w=self.weight) + + if self.predN and self.labelK: + for n in self.predN: + for k in self.labelK: + if n >= k: + metrics_to_compute[ + "group_recall_unweighted_at_L" + str(n) + "_at_H" + str(k) + ] = partial(recall_at_nk, n=n, k=k) + if self.weight: + metrics_to_compute[ + "group_recall_weighted_at_L" + str(n) + "_at_H" + str(k) + ] = partial(recall_at_nk, n=n, k=k, w=self.weight) + + if self.labelK and not self.predN: + for k in self.labelK: + metrics_to_compute["group_recall_unweighted_at_full_at_H" + str(k)] = partial( + recall_at_nk, k=k + ) + if self.weight: + metrics_to_compute["group_recall_weighted_at_full_at_H" + str(k)] = partial( + recall_at_nk, k=k, w=self.weight + ) + return metrics_to_compute + + def extract_label(self, prec, drec, drec_label): + return drec_label + + def extract_prediction(self, prec, drec, drec_label): + return extract_prediction_from_prediction_record(prec) diff --git a/pushservice/src/main/python/models/libs/metric_fn_utils.py b/pushservice/src/main/python/models/libs/metric_fn_utils.py new file mode 100644 index 0000000000..fc26a13052 --- /dev/null +++ b/pushservice/src/main/python/models/libs/metric_fn_utils.py @@ -0,0 +1,294 @@ +""" +Utilties for constructing a metric_fn for magic recs. +""" + +from twml.contrib.metrics.metrics import ( + get_dual_binary_tasks_metric_fn, + get_numeric_metric_fn, + get_partial_multi_binary_class_metric_fn, + get_single_binary_task_metric_fn, +) + +from .model_utils import generate_disliked_mask + +import tensorflow.compat.v1 as tf + + +METRIC_BOOK = { + "OONC": ["OONC"], + "OONC_Engagement": ["OONC", "Engagement"], + "Sent": ["Sent"], + "HeavyRankPosition": ["HeavyRankPosition"], + "HeavyRankProbability": ["HeavyRankProbability"], +} + +USER_AGE_FEATURE_NAME = "accountAge" +NEW_USER_AGE_CUTOFF = 0 + + +def remove_padding_and_flatten(tensor, valid_batch_size): + """Remove the padding of the input padded tensor given the valid batch size tensor, + then flatten the output with respect to the first dimension. + Args: + tensor: A tensor of size [META_BATCH_SIZE, BATCH_SIZE, FEATURE_DIM]. + valid_batch_size: A tensor of size [META_BATCH_SIZE], with each element indicating + the effective batch size of the BATCH_SIZE dimension. + + Returns: + A tesnor of size [tf.reduce_sum(valid_batch_size), FEATURE_DIM]. + """ + unpadded_ragged_tensor = tf.RaggedTensor.from_tensor(tensor=tensor, lengths=valid_batch_size) + + return unpadded_ragged_tensor.flat_values + + +def safe_mask(values, mask): + """Mask values if possible. + + Boolean mask inputed values if and only if values is a tensor of the same dimension as mask (or can be broadcasted to that dimension). + + Args: + values (Any or Tensor): Input tensor to mask. Dim 0 should be size N. + mask (boolean tensor): A boolean tensor of size N. + + Returns Values or Values masked. + """ + if values is None: + return values + if not tf.is_tensor(values): + return values + values_shape = values.get_shape() + if not values_shape or len(values_shape) == 0: + return values + if not mask.get_shape().is_compatible_with(values_shape[0]): + return values + return tf.boolean_mask(values, mask) + + +def add_new_user_metrics(metric_fn): + """Will stratify the metric_fn by adding new user metrics. + + Given an input metric_fn, double every metric: One will be the orignal and the other will only include those for new users. + + Args: + metric_fn (python function): Base twml metric_fn. + + Returns a metric_fn with new user metrics included. + """ + + def metric_fn_with_new_users(graph_output, labels, weights): + if USER_AGE_FEATURE_NAME not in graph_output: + raise ValueError( + "In order to get metrics stratified by user age, {name} feature should be added to model graph output. However, only the following output keys were found: {keys}.".format( + name=USER_AGE_FEATURE_NAME, keys=graph_output.keys() + ) + ) + + metric_ops = metric_fn(graph_output, labels, weights) + + is_new = tf.reshape( + tf.math.less_equal( + tf.cast(graph_output[USER_AGE_FEATURE_NAME], tf.int64), + tf.cast(NEW_USER_AGE_CUTOFF, tf.int64), + ), + [-1], + ) + + labels = safe_mask(labels, is_new) + weights = safe_mask(weights, is_new) + graph_output = {key: safe_mask(values, is_new) for key, values in graph_output.items()} + + new_user_metric_ops = metric_fn(graph_output, labels, weights) + new_user_metric_ops = {name + "_new_users": ops for name, ops in new_user_metric_ops.items()} + metric_ops.update(new_user_metric_ops) + return metric_ops + + return metric_fn_with_new_users + + +def get_meta_learn_single_binary_task_metric_fn( + metrics, classnames, top_k=(5, 5, 5), use_top_k=False +): + """Wrapper function to use the metric_fn with meta learning evaluation scheme. + + Args: + metrics: A list of string representing metric names. + classnames: A list of string repsenting class names, In case of multiple binary class models, + the names for each class or label. + top_k: A tuple of int to specify top K metrics. + use_top_k: A boolean value indicating of top K of metrics is used. + + Returns: + A customized metric_fn function. + """ + + def get_eval_metric_ops(graph_output, labels, weights): + """The op func of the eval_metrics. Comparing with normal version, + the difference is we flatten the output, label, and weights. + + Args: + graph_output: A dict of tensors. + labels: A tensor of int32 be the value of either 0 or 1. + weights: A tensor of float32 to indicate the per record weight. + + Returns: + A dict of metric names and values. + """ + metric_op_weighted = get_partial_multi_binary_class_metric_fn( + metrics, predcols=0, classes=classnames + ) + classnames_unweighted = ["unweighted_" + classname for classname in classnames] + metric_op_unweighted = get_partial_multi_binary_class_metric_fn( + metrics, predcols=0, classes=classnames_unweighted + ) + + valid_batch_size = graph_output["valid_batch_size"] + graph_output["output"] = remove_padding_and_flatten(graph_output["output"], valid_batch_size) + labels = remove_padding_and_flatten(labels, valid_batch_size) + weights = remove_padding_and_flatten(weights, valid_batch_size) + + tf.ensure_shape(graph_output["output"], [None, 1]) + tf.ensure_shape(labels, [None, 1]) + tf.ensure_shape(weights, [None, 1]) + + metrics_weighted = metric_op_weighted(graph_output, labels, weights) + metrics_unweighted = metric_op_unweighted(graph_output, labels, None) + metrics_weighted.update(metrics_unweighted) + + if use_top_k: + metric_op_numeric = get_numeric_metric_fn(metrics=None, topK=top_k, predcol=0, labelcol=1) + metrics_numeric = metric_op_numeric(graph_output, labels, weights) + metrics_weighted.update(metrics_numeric) + return metrics_weighted + + return get_eval_metric_ops + + +def get_meta_learn_dual_binary_tasks_metric_fn( + metrics, classnames, top_k=(5, 5, 5), use_top_k=False +): + """Wrapper function to use the metric_fn with meta learning evaluation scheme. + + Args: + metrics: A list of string representing metric names. + classnames: A list of string repsenting class names, In case of multiple binary class models, + the names for each class or label. + top_k: A tuple of int to specify top K metrics. + use_top_k: A boolean value indicating of top K of metrics is used. + + Returns: + A customized metric_fn function. + """ + + def get_eval_metric_ops(graph_output, labels, weights): + """The op func of the eval_metrics. Comparing with normal version, + the difference is we flatten the output, label, and weights. + + Args: + graph_output: A dict of tensors. + labels: A tensor of int32 be the value of either 0 or 1. + weights: A tensor of float32 to indicate the per record weight. + + Returns: + A dict of metric names and values. + """ + metric_op_weighted = get_partial_multi_binary_class_metric_fn( + metrics, predcols=[0, 1], classes=classnames + ) + classnames_unweighted = ["unweighted_" + classname for classname in classnames] + metric_op_unweighted = get_partial_multi_binary_class_metric_fn( + metrics, predcols=[0, 1], classes=classnames_unweighted + ) + + valid_batch_size = graph_output["valid_batch_size"] + graph_output["output"] = remove_padding_and_flatten(graph_output["output"], valid_batch_size) + labels = remove_padding_and_flatten(labels, valid_batch_size) + weights = remove_padding_and_flatten(weights, valid_batch_size) + + tf.ensure_shape(graph_output["output"], [None, 2]) + tf.ensure_shape(labels, [None, 2]) + tf.ensure_shape(weights, [None, 1]) + + metrics_weighted = metric_op_weighted(graph_output, labels, weights) + metrics_unweighted = metric_op_unweighted(graph_output, labels, None) + metrics_weighted.update(metrics_unweighted) + + if use_top_k: + metric_op_numeric = get_numeric_metric_fn(metrics=None, topK=top_k, predcol=2, labelcol=2) + metrics_numeric = metric_op_numeric(graph_output, labels, weights) + metrics_weighted.update(metrics_numeric) + return metrics_weighted + + return get_eval_metric_ops + + +def get_metric_fn(task_name, use_stratify_metrics, use_meta_batch=False): + """Will retrieve the metric_fn for magic recs. + + Args: + task_name (string): Which task is being used for this model. + use_stratify_metrics (boolean): Should we add stratified metrics (new user metrics). + use_meta_batch (boolean): If the output/label/weights are passed in 3D shape instead of + 2D shape. + + Returns: + A metric_fn function to pass in twml Trainer. + """ + if task_name not in METRIC_BOOK: + raise ValueError( + "Task name of {task_name} not recognized. Unable to retrieve metrics.".format( + task_name=task_name + ) + ) + class_names = METRIC_BOOK[task_name] + if use_meta_batch: + get_n_binary_task_metric_fn = ( + get_meta_learn_single_binary_task_metric_fn + if len(class_names) == 1 + else get_meta_learn_dual_binary_tasks_metric_fn + ) + else: + get_n_binary_task_metric_fn = ( + get_single_binary_task_metric_fn if len(class_names) == 1 else get_dual_binary_tasks_metric_fn + ) + + metric_fn = get_n_binary_task_metric_fn(metrics=None, classnames=METRIC_BOOK[task_name]) + + if use_stratify_metrics: + metric_fn = add_new_user_metrics(metric_fn) + + return metric_fn + + +def flip_disliked_labels(metric_fn): + """This function returns an adapted metric_fn which flips the labels of the OONCed evaluation data to 0 if it is disliked. + Args: + metric_fn: A metric_fn function to pass in twml Trainer. + + Returns: + _adapted_metric_fn: A customized metric_fn function with disliked OONC labels flipped. + """ + + def _adapted_metric_fn(graph_output, labels, weights): + """A customized metric_fn function with disliked OONC labels flipped. + + Args: + graph_output: A dict of tensors. + labels: labels of training samples, which is a 2D tensor of shape batch_size x 3: [OONCs, engagements, dislikes] + weights: A tensor of float32 to indicate the per record weight. + + Returns: + A dict of metric names and values. + """ + # We want to multiply the label of the observation by 0 only when it is disliked + disliked_mask = generate_disliked_mask(labels) + + # Extract OONC and engagement labels only. + labels = tf.reshape(labels[:, 0:2], shape=[-1, 2]) + + # Labels will be set to 0 if it is disliked. + adapted_labels = labels * tf.cast(tf.logical_not(disliked_mask), dtype=labels.dtype) + + return metric_fn(graph_output, adapted_labels, weights) + + return _adapted_metric_fn diff --git a/pushservice/src/main/python/models/libs/model_args.py b/pushservice/src/main/python/models/libs/model_args.py new file mode 100644 index 0000000000..ae142d818f --- /dev/null +++ b/pushservice/src/main/python/models/libs/model_args.py @@ -0,0 +1,231 @@ +from twml.trainers import DataRecordTrainer + + +# checkstyle: noqa + + +def get_arg_parser(): + parser = DataRecordTrainer.add_parser_arguments() + + parser.add_argument( + "--input_size_bits", + type=int, + default=18, + help="number of bits allocated to the input size", + ) + parser.add_argument( + "--model_trainer_name", + default="magic_recs_mlp_calibration_MTL_OONC_Engagement", + type=str, + help="specify the model trainer name.", + ) + + parser.add_argument( + "--model_type", + default="deepnorm_gbdt_inputdrop2_rescale", + type=str, + help="specify the model type to use.", + ) + parser.add_argument( + "--feat_config_type", + default="get_feature_config_with_sparse_continuous", + type=str, + help="specify the feature configure function to use.", + ) + + parser.add_argument( + "--directly_export_best", + default=False, + action="store_true", + help="whether to directly_export best_checkpoint", + ) + + parser.add_argument( + "--warm_start_base_dir", + default="none", + type=str, + help="latest ckpt in this folder will be used to ", + ) + + parser.add_argument( + "--feature_list", + default="none", + type=str, + help="Which features to use for training", + ) + parser.add_argument( + "--warm_start_from", default=None, type=str, help="model dir to warm start from" + ) + + parser.add_argument( + "--momentum", default=0.99999, type=float, help="Momentum term for batch normalization" + ) + parser.add_argument( + "--dropout", + default=0.2, + type=float, + help="input_dropout_rate to rescale output by (1 - input_dropout_rate)", + ) + parser.add_argument( + "--out_layer_1_size", default=256, type=int, help="Size of MLP_branch layer 1" + ) + parser.add_argument( + "--out_layer_2_size", default=128, type=int, help="Size of MLP_branch layer 2" + ) + parser.add_argument("--out_layer_3_size", default=64, type=int, help="Size of MLP_branch layer 3") + parser.add_argument( + "--sparse_embedding_size", default=50, type=int, help="Dimensionality of sparse embedding layer" + ) + parser.add_argument( + "--dense_embedding_size", default=128, type=int, help="Dimensionality of dense embedding layer" + ) + + parser.add_argument( + "--use_uam_label", + default=False, + type=str, + help="Whether to use uam_label or not", + ) + + parser.add_argument( + "--task_name", + default="OONC_Engagement", + type=str, + help="specify the task name to use: OONC or OONC_Engagement.", + ) + parser.add_argument( + "--init_weight", + default=0.9, + type=float, + help="Initial OONC Task Weight MTL: OONC+Engagement.", + ) + parser.add_argument( + "--use_engagement_weight", + default=False, + action="store_true", + help="whether to use engagement weight for base model.", + ) + parser.add_argument( + "--mtl_num_extra_layers", + type=int, + default=1, + help="Number of Hidden Layers for each TaskBranch.", + ) + parser.add_argument( + "--mtl_neuron_scale", type=int, default=4, help="Scaling Factor of Neurons in MTL Extra Layers." + ) + parser.add_argument( + "--use_oonc_score", + default=False, + action="store_true", + help="whether to use oonc score only or combined score.", + ) + parser.add_argument( + "--use_stratified_metrics", + default=False, + action="store_true", + help="Use stratified metrics: Break out new-user metrics.", + ) + parser.add_argument( + "--run_group_metrics", + default=False, + action="store_true", + help="Will run evaluation metrics grouped by user.", + ) + parser.add_argument( + "--use_full_scope", + default=False, + action="store_true", + help="Will add extra scope and naming to graph.", + ) + parser.add_argument( + "--trainable_regexes", + default=None, + nargs="*", + help="The union of variables specified by the list of regexes will be considered trainable.", + ) + parser.add_argument( + "--fine_tuning.ckpt_to_initialize_from", + dest="fine_tuning_ckpt_to_initialize_from", + type=str, + default=None, + help="Checkpoint path from which to warm start. Indicates the pre-trained model.", + ) + parser.add_argument( + "--fine_tuning.warm_start_scope_regex", + dest="fine_tuning_warm_start_scope_regex", + type=str, + default=None, + help="All variables matching this will be restored.", + ) + + return parser + + +def get_params(args=None): + parser = get_arg_parser() + if args is None: + return parser.parse_args() + else: + return parser.parse_args(args) + + +def get_arg_parser_light_ranking(): + parser = get_arg_parser() + + parser.add_argument( + "--use_record_weight", + default=False, + action="store_true", + help="whether to use record weight for base model.", + ) + parser.add_argument( + "--min_record_weight", default=0.0, type=float, help="Minimum record weight to use." + ) + parser.add_argument( + "--smooth_weight", default=0.0, type=float, help="Factor to smooth Rank Position Weight." + ) + + parser.add_argument( + "--num_mlp_layers", type=int, default=3, help="Number of Hidden Layers for MLP model." + ) + parser.add_argument( + "--mlp_neuron_scale", type=int, default=4, help="Scaling Factor of Neurons in MLP Layers." + ) + parser.add_argument( + "--run_light_ranking_group_metrics", + default=False, + action="store_true", + help="Will run evaluation metrics grouped by user for Light Ranking.", + ) + parser.add_argument( + "--use_missing_sub_branch", + default=False, + action="store_true", + help="Whether to use missing value sub-branch for Light Ranking.", + ) + parser.add_argument( + "--use_gbdt_features", + default=False, + action="store_true", + help="Whether to use GBDT features for Light Ranking.", + ) + parser.add_argument( + "--run_light_ranking_group_metrics_in_bq", + default=False, + action="store_true", + help="Whether to get_predictions for Light Ranking to compute group metrics in BigQuery.", + ) + parser.add_argument( + "--pred_file_path", + default=None, + type=str, + help="path", + ) + parser.add_argument( + "--pred_file_name", + default=None, + type=str, + help="path", + ) + return parser diff --git a/pushservice/src/main/python/models/libs/model_utils.py b/pushservice/src/main/python/models/libs/model_utils.py new file mode 100644 index 0000000000..1c53069118 --- /dev/null +++ b/pushservice/src/main/python/models/libs/model_utils.py @@ -0,0 +1,339 @@ +import sys + +import twml + +from .initializer import customized_glorot_uniform + +import tensorflow.compat.v1 as tf +import yaml + + +# checkstyle: noqa + + +def read_config(whitelist_yaml_file): + with tf.gfile.FastGFile(whitelist_yaml_file) as f: + try: + return yaml.safe_load(f) + except yaml.YAMLError as exc: + print(exc) + sys.exit(1) + + +def _sparse_feature_fixup(features, input_size_bits): + """Rebuild a sparse tensor feature so that its dense shape attribute is present. + + Arguments: + features (SparseTensor): Sparse feature tensor of shape ``(B, sparse_feature_dim)``. + input_size_bits (int): Number of columns in ``log2`` scale. Must be positive. + + Returns: + SparseTensor: Rebuilt and non-faulty version of `features`.""" + sparse_feature_dim = tf.constant(2**input_size_bits, dtype=tf.int64) + sparse_shape = tf.stack([features.dense_shape[0], sparse_feature_dim]) + sparse_tf = tf.SparseTensor(features.indices, features.values, sparse_shape) + return sparse_tf + + +def self_atten_dense(input, out_dim, activation=None, use_bias=True, name=None): + def safe_concat(base, suffix): + """Concats variables name components if base is given.""" + if not base: + return base + return f"{base}:{suffix}" + + input_dim = input.shape.as_list()[1] + + sigmoid_out = twml.layers.FullDense( + input_dim, dtype=tf.float32, activation=tf.nn.sigmoid, name=safe_concat(name, "sigmoid_out") + )(input) + atten_input = sigmoid_out * input + mlp_out = twml.layers.FullDense( + out_dim, + dtype=tf.float32, + activation=activation, + use_bias=use_bias, + name=safe_concat(name, "mlp_out"), + )(atten_input) + return mlp_out + + +def get_dense_out(input, out_dim, activation, dense_type): + if dense_type == "full_dense": + out = twml.layers.FullDense(out_dim, dtype=tf.float32, activation=activation)(input) + elif dense_type == "self_atten_dense": + out = self_atten_dense(input, out_dim, activation=activation) + return out + + +def get_input_trans_func(bn_normalized_dense, is_training): + gw_normalized_dense = tf.expand_dims(bn_normalized_dense, -1) + group_num = bn_normalized_dense.shape.as_list()[1] + + gw_normalized_dense = GroupWiseTrans(group_num, 1, 8, name="groupwise_1", activation=tf.tanh)( + gw_normalized_dense + ) + gw_normalized_dense = GroupWiseTrans(group_num, 8, 4, name="groupwise_2", activation=tf.tanh)( + gw_normalized_dense + ) + gw_normalized_dense = GroupWiseTrans(group_num, 4, 1, name="groupwise_3", activation=tf.tanh)( + gw_normalized_dense + ) + + gw_normalized_dense = tf.squeeze(gw_normalized_dense, [-1]) + + bn_gw_normalized_dense = tf.layers.batch_normalization( + gw_normalized_dense, + training=is_training, + renorm_momentum=0.9999, + momentum=0.9999, + renorm=is_training, + trainable=True, + ) + + return bn_gw_normalized_dense + + +def tensor_dropout( + input_tensor, + rate, + is_training, + sparse_tensor=None, +): + """ + Implements dropout layer for both dense and sparse input_tensor + + Arguments: + input_tensor: + B x D dense tensor, or a sparse tensor + rate (float32): + dropout rate + is_training (bool): + training stage or not. + sparse_tensor (bool): + whether the input_tensor is sparse tensor or not. Default to be None, this value has to be passed explicitly. + rescale_sparse_dropout (bool): + Do we need to do rescaling or not. + Returns: + tensor dropped out""" + if sparse_tensor == True: + if is_training: + with tf.variable_scope("sparse_dropout"): + values = input_tensor.values + keep_mask = tf.keras.backend.random_binomial( + tf.shape(values), p=1 - rate, dtype=tf.float32, seed=None + ) + keep_mask.set_shape([None]) + keep_mask = tf.cast(keep_mask, tf.bool) + + keep_indices = tf.boolean_mask(input_tensor.indices, keep_mask, axis=0) + keep_values = tf.boolean_mask(values, keep_mask, axis=0) + + dropped_tensor = tf.SparseTensor(keep_indices, keep_values, input_tensor.dense_shape) + return dropped_tensor + else: + return input_tensor + elif sparse_tensor == False: + return tf.layers.dropout(input_tensor, rate=rate, training=is_training) + + +def adaptive_transformation(bn_normalized_dense, is_training, func_type="default"): + assert func_type in [ + "default", + "tiny", + ], f"fun_type can only be one of default and tiny, but get {func_type}" + + gw_normalized_dense = tf.expand_dims(bn_normalized_dense, -1) + group_num = bn_normalized_dense.shape.as_list()[1] + + if func_type == "default": + gw_normalized_dense = FastGroupWiseTrans( + group_num, 1, 8, name="groupwise_1", activation=tf.tanh, init_multiplier=8 + )(gw_normalized_dense) + + gw_normalized_dense = FastGroupWiseTrans( + group_num, 8, 4, name="groupwise_2", activation=tf.tanh, init_multiplier=8 + )(gw_normalized_dense) + + gw_normalized_dense = FastGroupWiseTrans( + group_num, 4, 1, name="groupwise_3", activation=tf.tanh, init_multiplier=8 + )(gw_normalized_dense) + elif func_type == "tiny": + gw_normalized_dense = FastGroupWiseTrans( + group_num, 1, 2, name="groupwise_1", activation=tf.tanh, init_multiplier=8 + )(gw_normalized_dense) + + gw_normalized_dense = FastGroupWiseTrans( + group_num, 2, 1, name="groupwise_2", activation=tf.tanh, init_multiplier=8 + )(gw_normalized_dense) + + gw_normalized_dense = FastGroupWiseTrans( + group_num, 1, 1, name="groupwise_3", activation=tf.tanh, init_multiplier=8 + )(gw_normalized_dense) + + gw_normalized_dense = tf.squeeze(gw_normalized_dense, [-1]) + bn_gw_normalized_dense = tf.layers.batch_normalization( + gw_normalized_dense, + training=is_training, + renorm_momentum=0.9999, + momentum=0.9999, + renorm=is_training, + trainable=True, + ) + + return bn_gw_normalized_dense + + +class FastGroupWiseTrans(object): + """ + used to apply group-wise fully connected layers to the input. + it applies a tiny, unique MLP to each individual feature.""" + + def __init__(self, group_num, input_dim, out_dim, name, activation=None, init_multiplier=1): + self.group_num = group_num + self.input_dim = input_dim + self.out_dim = out_dim + self.activation = activation + self.init_multiplier = init_multiplier + + self.w = tf.get_variable( + name + "_group_weight", + [1, group_num, input_dim, out_dim], + initializer=customized_glorot_uniform( + fan_in=input_dim * init_multiplier, fan_out=out_dim * init_multiplier + ), + trainable=True, + ) + self.b = tf.get_variable( + name + "_group_bias", + [1, group_num, out_dim], + initializer=tf.constant_initializer(0.0), + trainable=True, + ) + + def __call__(self, input_tensor): + """ + input_tensor: batch_size x group_num x input_dim + output_tensor: batch_size x group_num x out_dim""" + input_tensor_expand = tf.expand_dims(input_tensor, axis=-1) + + output_tensor = tf.add( + tf.reduce_sum(tf.multiply(input_tensor_expand, self.w), axis=-2, keepdims=False), + self.b, + ) + + if self.activation is not None: + output_tensor = self.activation(output_tensor) + return output_tensor + + +class GroupWiseTrans(object): + """ + Used to apply group fully connected layers to the input. + """ + + def __init__(self, group_num, input_dim, out_dim, name, activation=None): + self.group_num = group_num + self.input_dim = input_dim + self.out_dim = out_dim + self.activation = activation + + w_list, b_list = [], [] + for idx in range(out_dim): + this_w = tf.get_variable( + name + f"_group_weight_{idx}", + [1, group_num, input_dim], + initializer=tf.keras.initializers.glorot_uniform(), + trainable=True, + ) + this_b = tf.get_variable( + name + f"_group_bias_{idx}", + [1, group_num, 1], + initializer=tf.constant_initializer(0.0), + trainable=True, + ) + w_list.append(this_w) + b_list.append(this_b) + self.w_list = w_list + self.b_list = b_list + + def __call__(self, input_tensor): + """ + input_tensor: batch_size x group_num x input_dim + output_tensor: batch_size x group_num x out_dim + """ + out_tensor_list = [] + for idx in range(self.out_dim): + this_res = ( + tf.reduce_sum(input_tensor * self.w_list[idx], axis=-1, keepdims=True) + self.b_list[idx] + ) + out_tensor_list.append(this_res) + output_tensor = tf.concat(out_tensor_list, axis=-1) + + if self.activation is not None: + output_tensor = self.activation(output_tensor) + return output_tensor + + +def add_scalar_summary(var, name, name_scope="hist_dense_feature/"): + with tf.name_scope("summaries/"): + with tf.name_scope(name_scope): + tf.summary.scalar(name, var) + + +def add_histogram_summary(var, name, name_scope="hist_dense_feature/"): + with tf.name_scope("summaries/"): + with tf.name_scope(name_scope): + tf.summary.histogram(name, tf.reshape(var, [-1])) + + +def sparse_clip_by_value(sparse_tf, min_val, max_val): + new_vals = tf.clip_by_value(sparse_tf.values, min_val, max_val) + return tf.SparseTensor(sparse_tf.indices, new_vals, sparse_tf.dense_shape) + + +def check_numerics_with_msg(tensor, message="", sparse_tensor=False): + if sparse_tensor: + values = tf.debugging.check_numerics(tensor.values, message=message) + return tf.SparseTensor(tensor.indices, values, tensor.dense_shape) + else: + return tf.debugging.check_numerics(tensor, message=message) + + +def pad_empty_sparse_tensor(tensor): + dummy_tensor = tf.SparseTensor( + indices=[[0, 0]], + values=[0.00001], + dense_shape=tensor.dense_shape, + ) + result = tf.cond( + tf.equal(tf.size(tensor.values), 0), + lambda: dummy_tensor, + lambda: tensor, + ) + return result + + +def filter_nans_and_infs(tensor, sparse_tensor=False): + if sparse_tensor: + sparse_values = tensor.values + filtered_val = tf.where( + tf.logical_or(tf.is_nan(sparse_values), tf.is_inf(sparse_values)), + tf.zeros_like(sparse_values), + sparse_values, + ) + return tf.SparseTensor(tensor.indices, filtered_val, tensor.dense_shape) + else: + return tf.where( + tf.logical_or(tf.is_nan(tensor), tf.is_inf(tensor)), tf.zeros_like(tensor), tensor + ) + + +def generate_disliked_mask(labels): + """Generate a disliked mask where only samples with dislike labels are set to 1 otherwise set to 0. + Args: + labels: labels of training samples, which is a 2D tensor of shape batch_size x 3: [OONCs, engagements, dislikes] + Returns: + 1D tensor of shape batch_size x 1: [dislikes (booleans)] + """ + return tf.equal(tf.reshape(labels[:, 2], shape=[-1, 1]), 1) diff --git a/pushservice/src/main/python/models/libs/warm_start_utils.py b/pushservice/src/main/python/models/libs/warm_start_utils.py new file mode 100644 index 0000000000..ca83df585d --- /dev/null +++ b/pushservice/src/main/python/models/libs/warm_start_utils.py @@ -0,0 +1,309 @@ +from collections import OrderedDict +import json +import os +from os.path import join + +from twitter.magicpony.common import file_access +import twml + +from .model_utils import read_config + +import numpy as np +from scipy import stats +import tensorflow.compat.v1 as tf + + +# checkstyle: noqa + + +def get_model_type_to_tensors_to_change_axis(): + model_type_to_tensors_to_change_axis = { + "magic_recs/model/batch_normalization/beta": ([0], "continuous"), + "magic_recs/model/batch_normalization/gamma": ([0], "continuous"), + "magic_recs/model/batch_normalization/moving_mean": ([0], "continuous"), + "magic_recs/model/batch_normalization/moving_stddev": ([0], "continuous"), + "magic_recs/model/batch_normalization/moving_variance": ([0], "continuous"), + "magic_recs/model/batch_normalization/renorm_mean": ([0], "continuous"), + "magic_recs/model/batch_normalization/renorm_stddev": ([0], "continuous"), + "magic_recs/model/logits/EngagementGivenOONC_logits/clem_net_1/block2_4/channel_wise_dense_4/kernel": ( + [1], + "all", + ), + "magic_recs/model/logits/OONC_logits/clem_net/block2/channel_wise_dense/kernel": ([1], "all"), + } + + return model_type_to_tensors_to_change_axis + + +def mkdirp(dirname): + if not tf.io.gfile.exists(dirname): + tf.io.gfile.makedirs(dirname) + + +def rename_dir(dirname, dst): + file_access.hdfs.mv(dirname, dst) + + +def rmdir(dirname): + if tf.io.gfile.exists(dirname): + if tf.io.gfile.isdir(dirname): + tf.io.gfile.rmtree(dirname) + else: + tf.io.gfile.remove(dirname) + + +def get_var_dict(checkpoint_path): + checkpoint = tf.train.get_checkpoint_state(checkpoint_path) + var_dict = OrderedDict() + with tf.Session() as sess: + all_var_list = tf.train.list_variables(checkpoint_path) + for var_name, _ in all_var_list: + # Load the variable + var = tf.train.load_variable(checkpoint_path, var_name) + var_dict[var_name] = var + return var_dict + + +def get_continunous_mapping_from_feat_list(old_feature_list, new_feature_list): + """ + get var_ind for old_feature and corresponding var_ind for new_feature + """ + new_var_ind, old_var_ind = [], [] + for this_new_id, this_new_name in enumerate(new_feature_list): + if this_new_name in old_feature_list: + this_old_id = old_feature_list.index(this_new_name) + new_var_ind.append(this_new_id) + old_var_ind.append(this_old_id) + return np.asarray(old_var_ind), np.asarray(new_var_ind) + + +def get_continuous_mapping_from_feat_dict(old_feature_dict, new_feature_dict): + """ + get var_ind for old_feature and corresponding var_ind for new_feature + """ + old_cont = old_feature_dict["continuous"] + old_bin = old_feature_dict["binary"] + + new_cont = new_feature_dict["continuous"] + new_bin = new_feature_dict["binary"] + + _dummy_sparse_feat = [f"sparse_feature_{_idx}" for _idx in range(100)] + + cont_old_var_ind, cont_new_var_ind = get_continunous_mapping_from_feat_list(old_cont, new_cont) + + all_old_var_ind, all_new_var_ind = get_continunous_mapping_from_feat_list( + old_cont + old_bin + _dummy_sparse_feat, new_cont + new_bin + _dummy_sparse_feat + ) + + _res = { + "continuous": (cont_old_var_ind, cont_new_var_ind), + "all": (all_old_var_ind, all_new_var_ind), + } + + return _res + + +def warm_start_from_var_dict( + old_ckpt_path, + var_ind_dict, + output_dir, + new_len_var, + var_to_change_dict_fn=get_model_type_to_tensors_to_change_axis, +): + """ + Parameters: + old_ckpt_path (str): path to the old checkpoint path + new_var_ind (array of int): index to overlapping features in new var between old and new feature list. + old_var_ind (array of int): index to overlapping features in old var between old and new feature list. + + output_dir (str): dir that used to write modified checkpoint + new_len_var ({str:int}): number of feature in the new feature list. + var_to_change_dict_fn (dict): A function to get the dictionary of format {var_name: dim_to_change} + """ + old_var_dict = get_var_dict(old_ckpt_path) + + ckpt_file_name = os.path.basename(old_ckpt_path) + mkdirp(output_dir) + output_path = join(output_dir, ckpt_file_name) + + tensors_to_change = var_to_change_dict_fn() + tf.compat.v1.reset_default_graph() + + with tf.Session() as sess: + var_name_shape_list = tf.train.list_variables(old_ckpt_path) + count = 0 + + for var_name, var_shape in var_name_shape_list: + old_var = old_var_dict[var_name] + if var_name in tensors_to_change.keys(): + _info_tuple = tensors_to_change[var_name] + dims_to_remove_from, var_type = _info_tuple + + new_var_ind, old_var_ind = var_ind_dict[var_type] + + this_shape = list(old_var.shape) + for this_dim in dims_to_remove_from: + this_shape[this_dim] = new_len_var[var_type] + + stddev = np.std(old_var) + truncated_norm_generator = stats.truncnorm(-0.5, 0.5, loc=0, scale=stddev) + size = np.prod(this_shape) + new_var = truncated_norm_generator.rvs(size).reshape(this_shape) + new_var = new_var.astype(old_var.dtype) + + new_var = copy_feat_based_on_mapping( + new_var, old_var, dims_to_remove_from, new_var_ind, old_var_ind + ) + count = count + 1 + else: + new_var = old_var + var = tf.Variable(new_var, name=var_name) + assert count == len(tensors_to_change.keys()), "not all variables are exchanged.\n" + saver = tf.train.Saver() + sess.run(tf.global_variables_initializer()) + saver.save(sess, output_path) + return output_path + + +def copy_feat_based_on_mapping(new_array, old_array, dims_to_remove_from, new_var_ind, old_var_ind): + if dims_to_remove_from == [0, 1]: + for this_new_ind, this_old_ind in zip(new_var_ind, old_var_ind): + new_array[this_new_ind, new_var_ind] = old_array[this_old_ind, old_var_ind] + elif dims_to_remove_from == [0]: + new_array[new_var_ind] = old_array[old_var_ind] + elif dims_to_remove_from == [1]: + new_array[:, new_var_ind] = old_array[:, old_var_ind] + else: + raise RuntimeError(f"undefined dims_to_remove_from pattern: ({dims_to_remove_from})") + return new_array + + +def read_file(filename, decode=False): + """ + Reads contents from a file and optionally decodes it. + + Arguments: + filename: + path to file where the contents will be loaded from. + Accepts HDFS and local paths. + decode: + False or 'json'. When decode='json', contents is decoded + with json.loads. When False, contents is returned as is. + """ + graph = tf.Graph() + with graph.as_default(): + read = tf.read_file(filename) + + with tf.Session(graph=graph) as sess: + contents = sess.run(read) + if not isinstance(contents, str): + contents = contents.decode() + + if decode == "json": + contents = json.loads(contents) + + return contents + + +def read_feat_list_from_disk(file_path): + return read_file(file_path, decode="json") + + +def get_feature_list_for_light_ranking(feature_list_path, data_spec_path): + feature_list = read_config(feature_list_path).items() + string_feat_list = [f[0] for f in feature_list if f[1] != "S"] + + feature_config_builder = twml.contrib.feature_config.FeatureConfigBuilder( + data_spec_path=data_spec_path + ) + feature_config_builder = feature_config_builder.extract_feature_group( + feature_regexes=string_feat_list, + group_name="continuous", + default_value=-1, + type_filter=["CONTINUOUS"], + ) + feature_config = feature_config_builder.build() + feature_list = feature_config_builder._feature_group_extraction_configs[0].feature_map[ + "CONTINUOUS" + ] + return feature_list + + +def get_feature_list_for_heavy_ranking(feature_list_path, data_spec_path): + feature_list = read_config(feature_list_path).items() + string_feat_list = [f[0] for f in feature_list if f[1] != "S"] + + feature_config_builder = twml.contrib.feature_config.FeatureConfigBuilder( + data_spec_path=data_spec_path + ) + feature_config_builder = feature_config_builder.extract_feature_group( + feature_regexes=string_feat_list, + group_name="continuous", + default_value=-1, + type_filter=["CONTINUOUS"], + ) + + feature_config_builder = feature_config_builder.extract_feature_group( + feature_regexes=string_feat_list, + group_name="binary", + default_value=False, + type_filter=["BINARY"], + ) + + feature_config_builder = feature_config_builder.build() + + continuous_feature_list = feature_config_builder._feature_group_extraction_configs[0].feature_map[ + "CONTINUOUS" + ] + + binary_feature_list = feature_config_builder._feature_group_extraction_configs[1].feature_map[ + "BINARY" + ] + return {"continuous": continuous_feature_list, "binary": binary_feature_list} + + +def warm_start_checkpoint( + old_best_ckpt_folder, + old_feature_list_path, + feature_allow_list_path, + data_spec_path, + output_ckpt_folder, + *args, +): + """ + Reads old checkpoint and the old feature list, and create a new ckpt warm started from old ckpt using new features . + + Arguments: + old_best_ckpt_folder: + path to the best_checkpoint_folder for old model + old_feature_list_path: + path to the json file that stores the list of continuous features used in old models. + feature_allow_list_path: + yaml file that contain the feature allow list. + data_spec_path: + path to the data_spec file + output_ckpt_folder: + folder that contains the modified ckpt. + + Returns: + path to the modified ckpt.""" + old_ckpt_path = tf.train.latest_checkpoint(old_best_ckpt_folder, latest_filename=None) + + new_feature_dict = get_feature_list(feature_allow_list_path, data_spec_path) + old_feature_dict = read_feat_list_from_disk(old_feature_list_path) + + var_ind_dict = get_continuous_mapping_from_feat_dict(new_feature_dict, old_feature_dict) + + new_len_var = { + "continuous": len(new_feature_dict["continuous"]), + "all": len(new_feature_dict["continuous"] + new_feature_dict["binary"]) + 100, + } + + warm_started_ckpt_path = warm_start_from_var_dict( + old_ckpt_path, + var_ind_dict, + output_dir=output_ckpt_folder, + new_len_var=new_len_var, + ) + + return warm_started_ckpt_path diff --git a/pushservice/src/main/python/models/light_ranking/BUILD b/pushservice/src/main/python/models/light_ranking/BUILD new file mode 100644 index 0000000000..e88d7de7cf --- /dev/null +++ b/pushservice/src/main/python/models/light_ranking/BUILD @@ -0,0 +1,69 @@ +#":mlwf_libs", + +python37_binary( + name = "eval_model", + source = "eval_model.py", + dependencies = [ + ":libs", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:eval_model", + ], +) + +python37_binary( + name = "train_model", + source = "deep_norm.py", + dependencies = [ + ":libs", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:train_model", + ], +) + +python37_binary( + name = "train_model_local", + source = "deep_norm.py", + dependencies = [ + ":libs", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:train_model_local", + "twml", + ], +) + +python37_binary( + name = "eval_model_local", + source = "eval_model.py", + dependencies = [ + ":libs", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:eval_model_local", + "twml", + ], +) + +python37_binary( + name = "mlwf_model", + source = "deep_norm.py", + dependencies = [ + ":mlwf_libs", + "3rdparty/python/_closures/frigate/frigate-pushservice-opensource/src/main/python/models/light_ranking:mlwf_model", + ], +) + +python3_library( + name = "libs", + sources = ["**/*.py"], + tags = ["no-mypy"], + dependencies = [ + "src/python/twitter/deepbird/projects/magic_recs/libs", + "src/python/twitter/deepbird/util/data", + "twml:twml-nodeps", + ], +) + +python3_library( + name = "mlwf_libs", + sources = ["**/*.py"], + tags = ["no-mypy"], + dependencies = [ + "src/python/twitter/deepbird/projects/magic_recs/libs", + "twml", + ], +) diff --git a/pushservice/src/main/python/models/light_ranking/README.md b/pushservice/src/main/python/models/light_ranking/README.md new file mode 100644 index 0000000000..9d7bd2682c --- /dev/null +++ b/pushservice/src/main/python/models/light_ranking/README.md @@ -0,0 +1,14 @@ +# Notification Light Ranker Model + +## Model Context +There are 4 major components of Twitter notifications recommendation system: 1) candidate generation 2) light ranking 3) heavy ranking & 4) quality control. This notification light ranker model bridges candidate generation and heavy ranking by pre-selecting highly-relevant candidates from the initial huge candidate pool. It’s a light-weight model to reduce system cost during heavy ranking without hurting user experience. + +## Directory Structure +- BUILD: this file defines python library dependencies +- model_pools_mlp.py: this file defines tensorflow model architecture for the notification light ranker model +- deep_norm.py: this file contains 1) how to build the tensorflow graph with specified model architecture, loss function and training configuration. 2) how to set up the overall model training & evaluation pipeline +- eval_model.py: the main python entry file to set up the overall model evaluation pipeline + + + + diff --git a/pushservice/src/main/python/models/light_ranking/__init__.py b/pushservice/src/main/python/models/light_ranking/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pushservice/src/main/python/models/light_ranking/deep_norm.py b/pushservice/src/main/python/models/light_ranking/deep_norm.py new file mode 100644 index 0000000000..bc90deba43 --- /dev/null +++ b/pushservice/src/main/python/models/light_ranking/deep_norm.py @@ -0,0 +1,226 @@ +from datetime import datetime +from functools import partial +import os + +from twitter.cortex.ml.embeddings.common.helpers import decode_str_or_unicode +import twml +from twml.trainers import DataRecordTrainer + +from ..libs.get_feat_config import get_feature_config_light_ranking, LABELS_LR +from ..libs.graph_utils import get_trainable_variables +from ..libs.group_metrics import ( + run_group_metrics_light_ranking, + run_group_metrics_light_ranking_in_bq, +) +from ..libs.metric_fn_utils import get_metric_fn +from ..libs.model_args import get_arg_parser_light_ranking +from ..libs.model_utils import read_config +from ..libs.warm_start_utils import get_feature_list_for_light_ranking +from .model_pools_mlp import light_ranking_mlp_ngbdt + +import tensorflow.compat.v1 as tf +from tensorflow.compat.v1 import logging + + +# checkstyle: noqa + + +def build_graph( + features, label, mode, params, config=None, run_light_ranking_group_metrics_in_bq=False +): + is_training = mode == tf.estimator.ModeKeys.TRAIN + this_model_func = light_ranking_mlp_ngbdt + model_output = this_model_func(features, is_training, params, label) + + logits = model_output["output"] + graph_output = {} + # -------------------------------------------------------- + # define graph output dict + # -------------------------------------------------------- + if mode == tf.estimator.ModeKeys.PREDICT: + loss = None + output_label = "prediction" + if params.task_name in LABELS_LR: + output = tf.nn.sigmoid(logits) + output = tf.clip_by_value(output, 0, 1) + + if run_light_ranking_group_metrics_in_bq: + graph_output["trace_id"] = features["meta.trace_id"] + graph_output["target"] = features["meta.ranking.weighted_oonc_model_score"] + + else: + raise ValueError("Invalid Task Name !") + + else: + output_label = "output" + weights = tf.cast(features["weights"], dtype=tf.float32, name="RecordWeights") + + if params.task_name in LABELS_LR: + if params.use_record_weight: + weights = tf.clip_by_value( + 1.0 / (1.0 + weights + params.smooth_weight), params.min_record_weight, 1.0 + ) + + loss = tf.reduce_sum( + tf.nn.sigmoid_cross_entropy_with_logits(labels=label, logits=logits) * weights + ) / (tf.reduce_sum(weights)) + else: + loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=label, logits=logits)) + output = tf.nn.sigmoid(logits) + + else: + raise ValueError("Invalid Task Name !") + + train_op = None + if mode == tf.estimator.ModeKeys.TRAIN: + # -------------------------------------------------------- + # get train_op + # -------------------------------------------------------- + optimizer = tf.train.GradientDescentOptimizer(learning_rate=params.learning_rate) + update_ops = set(tf.get_collection(tf.GraphKeys.UPDATE_OPS)) + variables = get_trainable_variables( + all_trainable_variables=tf.trainable_variables(), trainable_regexes=params.trainable_regexes + ) + with tf.control_dependencies(update_ops): + train_op = twml.optimizers.optimize_loss( + loss=loss, + variables=variables, + global_step=tf.train.get_global_step(), + optimizer=optimizer, + learning_rate=params.learning_rate, + learning_rate_decay_fn=twml.learning_rate_decay.get_learning_rate_decay_fn(params), + ) + + graph_output[output_label] = output + graph_output["loss"] = loss + graph_output["train_op"] = train_op + return graph_output + + +def get_params(args=None): + parser = get_arg_parser_light_ranking() + if args is None: + return parser.parse_args() + else: + return parser.parse_args(args) + + +def _main(): + opt = get_params() + logging.info("parse is: ") + logging.info(opt) + + feature_list = read_config(opt.feature_list).items() + feature_config = get_feature_config_light_ranking( + data_spec_path=opt.data_spec, + feature_list_provided=feature_list, + opt=opt, + add_gbdt=opt.use_gbdt_features, + run_light_ranking_group_metrics_in_bq=opt.run_light_ranking_group_metrics_in_bq, + ) + feature_list_path = opt.feature_list + + # -------------------------------------------------------- + # Create Trainer + # -------------------------------------------------------- + trainer = DataRecordTrainer( + name=opt.model_trainer_name, + params=opt, + build_graph_fn=build_graph, + save_dir=opt.save_dir, + run_config=None, + feature_config=feature_config, + metric_fn=get_metric_fn(opt.task_name, use_stratify_metrics=False), + ) + if opt.directly_export_best: + logging.info("Directly exporting the model without training") + else: + # ---------------------------------------------------- + # Model Training & Evaluation + # ---------------------------------------------------- + eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) + train_input_fn = trainer.get_train_input_fn(shuffle=True) + + if opt.distributed or opt.num_workers is not None: + learn = trainer.train_and_evaluate + else: + learn = trainer.learn + logging.info("Training...") + start = datetime.now() + + early_stop_metric = "rce_unweighted_" + opt.task_name + learn( + early_stop_minimize=False, + early_stop_metric=early_stop_metric, + early_stop_patience=opt.early_stop_patience, + early_stop_tolerance=opt.early_stop_tolerance, + eval_input_fn=eval_input_fn, + train_input_fn=train_input_fn, + ) + + end = datetime.now() + logging.info("Training time: " + str(end - start)) + + logging.info("Exporting the models...") + + # -------------------------------------------------------- + # Do the model exporting + # -------------------------------------------------------- + start = datetime.now() + if not opt.export_dir: + opt.export_dir = os.path.join(opt.save_dir, "exported_models") + + raw_model_path = twml.contrib.export.export_fn.export_all_models( + trainer=trainer, + export_dir=opt.export_dir, + parse_fn=feature_config.get_parse_fn(), + serving_input_receiver_fn=feature_config.get_serving_input_receiver_fn(), + export_output_fn=twml.export_output_fns.batch_prediction_continuous_output_fn, + ) + export_model_dir = decode_str_or_unicode(raw_model_path) + + logging.info("Model export time: " + str(datetime.now() - start)) + logging.info("The saved model directory is: " + opt.save_dir) + + tf.logging.info("getting default continuous_feature_list") + continuous_feature_list = get_feature_list_for_light_ranking(feature_list_path, opt.data_spec) + continous_feature_list_save_path = os.path.join(opt.save_dir, "continuous_feature_list.json") + twml.util.write_file(continous_feature_list_save_path, continuous_feature_list, encode="json") + tf.logging.info(f"Finish writting files to {continous_feature_list_save_path}") + + if opt.run_light_ranking_group_metrics: + # -------------------------------------------- + # Run Light Ranking Group Metrics + # -------------------------------------------- + run_group_metrics_light_ranking( + trainer=trainer, + data_dir=os.path.join(opt.eval_data_dir, opt.eval_start_datetime), + model_path=export_model_dir, + parse_fn=feature_config.get_parse_fn(), + ) + + if opt.run_light_ranking_group_metrics_in_bq: + # ---------------------------------------------------------------------------------------- + # Get Light/Heavy Ranker Predictions for Light Ranking Group Metrics in BigQuery + # ---------------------------------------------------------------------------------------- + trainer_pred = DataRecordTrainer( + name=opt.model_trainer_name, + params=opt, + build_graph_fn=partial(build_graph, run_light_ranking_group_metrics_in_bq=True), + save_dir=opt.save_dir + "/tmp/", + run_config=None, + feature_config=feature_config, + metric_fn=get_metric_fn(opt.task_name, use_stratify_metrics=False), + ) + checkpoint_folder = os.path.join(opt.save_dir, "best_checkpoint") + checkpoint = tf.train.latest_checkpoint(checkpoint_folder, latest_filename=None) + tf.logging.info("\n\nPrediction from Checkpoint: {:}.\n\n".format(checkpoint)) + run_group_metrics_light_ranking_in_bq( + trainer=trainer_pred, params=opt, checkpoint_path=checkpoint + ) + + tf.logging.info("Done Training & Prediction.") + + +if __name__ == "__main__": + _main() diff --git a/pushservice/src/main/python/models/light_ranking/eval_model.py b/pushservice/src/main/python/models/light_ranking/eval_model.py new file mode 100644 index 0000000000..1726685cff --- /dev/null +++ b/pushservice/src/main/python/models/light_ranking/eval_model.py @@ -0,0 +1,89 @@ +from datetime import datetime +from functools import partial +import os + +from ..libs.group_metrics import ( + run_group_metrics_light_ranking, + run_group_metrics_light_ranking_in_bq, +) +from ..libs.metric_fn_utils import get_metric_fn +from ..libs.model_args import get_arg_parser_light_ranking +from ..libs.model_utils import read_config +from .deep_norm import build_graph, DataRecordTrainer, get_config_func, logging + + +# checkstyle: noqa + +if __name__ == "__main__": + parser = get_arg_parser_light_ranking() + parser.add_argument( + "--eval_checkpoint", + default=None, + type=str, + help="Which checkpoint to use for evaluation", + ) + parser.add_argument( + "--saved_model_path", + default=None, + type=str, + help="Path to saved model for evaluation", + ) + parser.add_argument( + "--run_binary_metrics", + default=False, + action="store_true", + help="Whether to compute the basic binary metrics for Light Ranking.", + ) + + opt = parser.parse_args() + logging.info("parse is: ") + logging.info(opt) + + feature_list = read_config(opt.feature_list).items() + feature_config = get_config_func(opt.feat_config_type)( + data_spec_path=opt.data_spec, + feature_list_provided=feature_list, + opt=opt, + add_gbdt=opt.use_gbdt_features, + run_light_ranking_group_metrics_in_bq=opt.run_light_ranking_group_metrics_in_bq, + ) + + # ----------------------------------------------- + # Create Trainer + # ----------------------------------------------- + trainer = DataRecordTrainer( + name=opt.model_trainer_name, + params=opt, + build_graph_fn=partial(build_graph, run_light_ranking_group_metrics_in_bq=True), + save_dir=opt.save_dir, + run_config=None, + feature_config=feature_config, + metric_fn=get_metric_fn(opt.task_name, use_stratify_metrics=False), + ) + + # ----------------------------------------------- + # Model Evaluation + # ----------------------------------------------- + logging.info("Evaluating...") + start = datetime.now() + + if opt.run_binary_metrics: + eval_input_fn = trainer.get_eval_input_fn(repeat=False, shuffle=False) + eval_steps = None if (opt.eval_steps is not None and opt.eval_steps < 0) else opt.eval_steps + trainer.estimator.evaluate(eval_input_fn, steps=eval_steps, checkpoint_path=opt.eval_checkpoint) + + if opt.run_light_ranking_group_metrics_in_bq: + run_group_metrics_light_ranking_in_bq( + trainer=trainer, params=opt, checkpoint_path=opt.eval_checkpoint + ) + + if opt.run_light_ranking_group_metrics: + run_group_metrics_light_ranking( + trainer=trainer, + data_dir=os.path.join(opt.eval_data_dir, opt.eval_start_datetime), + model_path=opt.saved_model_path, + parse_fn=feature_config.get_parse_fn(), + ) + + end = datetime.now() + logging.info("Evaluating time: " + str(end - start)) diff --git a/pushservice/src/main/python/models/light_ranking/model_pools_mlp.py b/pushservice/src/main/python/models/light_ranking/model_pools_mlp.py new file mode 100644 index 0000000000..b45c85e470 --- /dev/null +++ b/pushservice/src/main/python/models/light_ranking/model_pools_mlp.py @@ -0,0 +1,187 @@ +import warnings + +from twml.contrib.layers import ZscoreNormalization + +from ...libs.customized_full_sparse import FullSparse +from ...libs.get_feat_config import FEAT_CONFIG_DEFAULT_VAL as MISSING_VALUE_MARKER +from ...libs.model_utils import ( + _sparse_feature_fixup, + adaptive_transformation, + filter_nans_and_infs, + get_dense_out, + tensor_dropout, +) + +import tensorflow.compat.v1 as tf +# checkstyle: noqa + +def light_ranking_mlp_ngbdt(features, is_training, params, label=None): + return deepnorm_light_ranking( + features, + is_training, + params, + label=label, + decay=params.momentum, + dense_emb_size=params.dense_embedding_size, + base_activation=tf.keras.layers.LeakyReLU(), + input_dropout_rate=params.dropout, + use_gbdt=False, + ) + + +def deepnorm_light_ranking( + features, + is_training, + params, + label=None, + decay=0.99999, + dense_emb_size=128, + base_activation=None, + input_dropout_rate=None, + input_dense_type="self_atten_dense", + emb_dense_type="self_atten_dense", + mlp_dense_type="self_atten_dense", + use_gbdt=False, +): + # -------------------------------------------------------- + # Initial Parameter Checking + # -------------------------------------------------------- + if base_activation is None: + base_activation = tf.keras.layers.LeakyReLU() + + if label is not None: + warnings.warn( + "Label is unused in deepnorm_gbdt. Stop using this argument.", + DeprecationWarning, + ) + + with tf.variable_scope("helper_layers"): + full_sparse_layer = FullSparse( + output_size=params.sparse_embedding_size, + activation=base_activation, + use_sparse_grads=is_training, + use_binary_values=False, + dtype=tf.float32, + ) + input_normalizing_layer = ZscoreNormalization(decay=decay, name="input_normalizing_layer") + + # -------------------------------------------------------- + # Feature Selection & Embedding + # -------------------------------------------------------- + if use_gbdt: + sparse_gbdt_features = _sparse_feature_fixup(features["gbdt_sparse"], params.input_size_bits) + if input_dropout_rate is not None: + sparse_gbdt_features = tensor_dropout( + sparse_gbdt_features, input_dropout_rate, is_training, sparse_tensor=True + ) + + total_embed = full_sparse_layer(sparse_gbdt_features, use_binary_values=True) + + if (input_dropout_rate is not None) and is_training: + total_embed = total_embed / (1 - input_dropout_rate) + + else: + with tf.variable_scope("dense_branch"): + dense_continuous_features = filter_nans_and_infs(features["continuous"]) + + if params.use_missing_sub_branch: + is_missing = tf.equal(dense_continuous_features, MISSING_VALUE_MARKER) + continuous_features_filled = tf.where( + is_missing, + tf.zeros_like(dense_continuous_features), + dense_continuous_features, + ) + normalized_features = input_normalizing_layer( + continuous_features_filled, is_training, tf.math.logical_not(is_missing) + ) + + with tf.variable_scope("missing_sub_branch"): + missing_feature_embed = get_dense_out( + tf.cast(is_missing, tf.float32), + dense_emb_size, + activation=base_activation, + dense_type=input_dense_type, + ) + + else: + continuous_features_filled = dense_continuous_features + normalized_features = input_normalizing_layer(continuous_features_filled, is_training) + + with tf.variable_scope("continuous_sub_branch"): + normalized_features = adaptive_transformation( + normalized_features, is_training, func_type="tiny" + ) + + if input_dropout_rate is not None: + normalized_features = tensor_dropout( + normalized_features, + input_dropout_rate, + is_training, + sparse_tensor=False, + ) + filled_feature_embed = get_dense_out( + normalized_features, + dense_emb_size, + activation=base_activation, + dense_type=input_dense_type, + ) + + if params.use_missing_sub_branch: + dense_embed = tf.concat( + [filled_feature_embed, missing_feature_embed], axis=1, name="merge_dense_emb" + ) + else: + dense_embed = filled_feature_embed + + with tf.variable_scope("sparse_branch"): + sparse_discrete_features = _sparse_feature_fixup( + features["sparse_no_continuous"], params.input_size_bits + ) + if input_dropout_rate is not None: + sparse_discrete_features = tensor_dropout( + sparse_discrete_features, input_dropout_rate, is_training, sparse_tensor=True + ) + + discrete_features_embed = full_sparse_layer(sparse_discrete_features, use_binary_values=True) + + if (input_dropout_rate is not None) and is_training: + discrete_features_embed = discrete_features_embed / (1 - input_dropout_rate) + + total_embed = tf.concat( + [dense_embed, discrete_features_embed], + axis=1, + name="total_embed", + ) + + total_embed = tf.layers.batch_normalization( + total_embed, + training=is_training, + renorm_momentum=decay, + momentum=decay, + renorm=is_training, + trainable=True, + ) + + # -------------------------------------------------------- + # MLP Layers + # -------------------------------------------------------- + with tf.variable_scope("MLP_branch"): + + assert params.num_mlp_layers >= 0 + embed_list = [total_embed] + [None for _ in range(params.num_mlp_layers)] + dense_types = [emb_dense_type] + [mlp_dense_type for _ in range(params.num_mlp_layers - 1)] + + for xl in range(1, params.num_mlp_layers + 1): + neurons = params.mlp_neuron_scale ** (params.num_mlp_layers + 1 - xl) + embed_list[xl] = get_dense_out( + embed_list[xl - 1], neurons, activation=base_activation, dense_type=dense_types[xl - 1] + ) + + if params.task_name in ["Sent", "HeavyRankPosition", "HeavyRankProbability"]: + logits = get_dense_out(embed_list[-1], 1, activation=None, dense_type=mlp_dense_type) + + else: + raise ValueError("Invalid Task Name !") + + output_dict = {"output": logits} + return output_dict diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.bazel b/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.bazel new file mode 100644 index 0000000000..d53d4e2518 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/BUILD.bazel @@ -0,0 +1,337 @@ +scala_library( + sources = ["**/*.scala"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + tags = [ + "bazel-compatible", + ], + dependencies = [ + "3rdparty/jvm/com/twitter/bijection:scrooge", + "3rdparty/jvm/com/twitter/storehaus:core", + "abdecider", + "abuse/detection/src/main/thrift/com/twitter/abuse/detection/scoring:thrift-scala", + "ann/src/main/scala/com/twitter/ann/common", + "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", + "audience-rewards/thrift/src/main/thrift:thrift-scala", + "communities/thrift/src/main/thrift/com/twitter/communities:thrift-scala", + "configapi/configapi-core", + "configapi/configapi-decider", + "content-mixer/thrift/src/main/thrift:thrift-scala", + "content-recommender/thrift/src/main/thrift:thrift-scala", + "copyselectionservice/server/src/main/scala/com/twitter/copyselectionservice/algorithms", + "copyselectionservice/thrift/src/main/thrift:copyselectionservice-scala", + "cortex-deepbird/thrift/src/main/thrift:thrift-java", + "cr-mixer/thrift/src/main/thrift:thrift-scala", + "cuad/projects/hashspace/thrift:thrift-scala", + "cuad/projects/tagspace/thrift/src/main/thrift:thrift-scala", + "detopic/thrift/src/main/thrift:thrift-scala", + "discovery-common/src/main/scala/com/twitter/discovery/common/configapi", + "discovery-common/src/main/scala/com/twitter/discovery/common/ddg", + "discovery-common/src/main/scala/com/twitter/discovery/common/environment", + "discovery-common/src/main/scala/com/twitter/discovery/common/fatigue", + "discovery-common/src/main/scala/com/twitter/discovery/common/nackwarmupfilter", + "discovery-common/src/main/scala/com/twitter/discovery/common/server", + "discovery-ds/src/main/thrift/com/twitter/dds/scio/searcher_aggregate_history_srp:searcher_aggregate_history_srp-scala", + "escherbird/src/scala/com/twitter/escherbird/util/metadatastitch", + "escherbird/src/scala/com/twitter/escherbird/util/uttclient", + "escherbird/src/thrift/com/twitter/escherbird/utt:strato-columns-scala", + "eventbus/client", + "eventdetection/event_context/src/main/scala/com/twitter/eventdetection/event_context/util", + "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", + "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", + "featureswitches/featureswitches-core/src/main/scala", + "featureswitches/featureswitches-core/src/main/scala:dynmap", + "featureswitches/featureswitches-core/src/main/scala:recipient", + "featureswitches/featureswitches-core/src/main/scala:useragent", + "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", + "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", + "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/server", + "finagle-internal/ostrich-stats", + "finagle/finagle-core/src/main", + "finagle/finagle-http/src/main/scala", + "finagle/finagle-memcached/src/main/scala", + "finagle/finagle-stats", + "finagle/finagle-thriftmux", + "finagle/finagle-tunable/src/main/scala", + "finagle/finagle-zipkin-scribe", + "finatra-internal/abdecider", + "finatra-internal/decider", + "finatra-internal/mtls-http/src/main/scala", + "finatra-internal/mtls-thriftmux/src/main/scala", + "finatra/http-client/src/main/scala", + "finatra/http-core/src/main/java/com/twitter/finatra/http", + "finatra/http-core/src/main/scala/com/twitter/finatra/http/response", + "finatra/http-server/src/main/scala/com/twitter/finatra/http", + "finatra/http-server/src/main/scala/com/twitter/finatra/http/filters", + "finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations", + "finatra/inject/inject-app/src/main/scala", + "finatra/inject/inject-core/src/main/scala", + "finatra/inject/inject-server/src/main/scala", + "finatra/inject/inject-slf4j/src/main/scala/com/twitter/inject", + "finatra/inject/inject-thrift-client/src/main/scala", + "finatra/inject/inject-utils/src/main/scala", + "finatra/utils/src/main/java/com/twitter/finatra/annotations", + "fleets/fleets-proxy/thrift/src/main/thrift:fleet-scala", + "fleets/fleets-proxy/thrift/src/main/thrift/service:baseservice-scala", + "flock-client/src/main/scala", + "flock-client/src/main/thrift:thrift-scala", + "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", + "frigate/frigate-common:base", + "frigate/frigate-common:config", + "frigate/frigate-common:debug", + "frigate/frigate-common:entity_graph_client", + "frigate/frigate-common:history", + "frigate/frigate-common:logger", + "frigate/frigate-common:ml-base", + "frigate/frigate-common:ml-feature", + "frigate/frigate-common:ml-prediction", + "frigate/frigate-common:ntab", + "frigate/frigate-common:predicate", + "frigate/frigate-common:rec_types", + "frigate/frigate-common:score_summary", + "frigate/frigate-common:util", + "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/candidate", + "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/experiments", + "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/filter", + "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/modules/store:semantic_core_stores", + "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store", + "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/deviceinfo", + "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests", + "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", + "frigate/push-mixer/thrift/src/main/thrift:thrift-scala", + "geo/geo-prediction/src/main/thrift:local-viral-tweets-thrift-scala", + "geoduck/service/src/main/scala/com/twitter/geoduck/service/common/clientmodules", + "geoduck/util/country", + "gizmoduck/client/src/main/scala/com/twitter/gizmoduck/testusers/client", + "hermit/hermit-core:model-user_state", + "hermit/hermit-core:predicate", + "hermit/hermit-core:predicate-gizmoduck", + "hermit/hermit-core:predicate-scarecrow", + "hermit/hermit-core:predicate-socialgraph", + "hermit/hermit-core:predicate-tweetypie", + "hermit/hermit-core:store-labeled_push_recs", + "hermit/hermit-core:store-metastore", + "hermit/hermit-core:store-timezone", + "hermit/hermit-core:store-tweetypie", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/gizmoduck", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/scarecrow", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/semantic_core", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/user_htl_session_store", + "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/user_interest", + "hmli/hss/src/main/thrift/com/twitter/hss:thrift-scala", + "ibis2/service/src/main/scala/com/twitter/ibis2/lib", + "ibis2/service/src/main/thrift/com/twitter/ibis2/service:ibis2-service-scala", + "interests-service/thrift/src/main/thrift:thrift-scala", + "interests_discovery/thrift/src/main/thrift:batch-thrift-scala", + "interests_discovery/thrift/src/main/thrift:service-thrift-scala", + "kujaku/thrift/src/main/thrift:domain-scala", + "live-video-timeline/client/src/main/scala/com/twitter/livevideo/timeline/client/v2", + "live-video-timeline/domain/src/main/scala/com/twitter/livevideo/timeline/domain", + "live-video-timeline/domain/src/main/scala/com/twitter/livevideo/timeline/domain/v2", + "live-video-timeline/thrift/src/main/thrift/com/twitter/livevideo/timeline:thrift-scala", + "live-video/common/src/main/scala/com/twitter/livevideo/common/domain/v2", + "live-video/common/src/main/scala/com/twitter/livevideo/common/ids", + "notifications-platform/inbound-notifications/src/main/thrift/com/twitter/inbound_notifications:exception-scala", + "notifications-platform/inbound-notifications/src/main/thrift/com/twitter/inbound_notifications:thrift-scala", + "notifications-platform/platform-lib/src/main/thrift/com/twitter/notifications/platform:custom-notification-actions-scala", + "notifications-platform/platform-lib/src/main/thrift/com/twitter/notifications/platform:thrift-scala", + "notifications-relevance/src/scala/com/twitter/nrel/heavyranker", + "notifications-relevance/src/scala/com/twitter/nrel/hydration/base", + "notifications-relevance/src/scala/com/twitter/nrel/hydration/frigate", + "notifications-relevance/src/scala/com/twitter/nrel/hydration/push", + "notifications-relevance/src/scala/com/twitter/nrel/lightranker", + "notificationservice/common/src/main/scala/com/twitter/notificationservice/genericfeedbackstore", + "notificationservice/common/src/main/scala/com/twitter/notificationservice/model:alias", + "notificationservice/common/src/main/scala/com/twitter/notificationservice/model/service", + "notificationservice/common/src/test/scala/com/twitter/notificationservice/mocks", + "notificationservice/scribe/src/main/scala/com/twitter/notificationservice/scribe/manhattan:mh_wrapper", + "notificationservice/thrift/src/main/thrift/com/twitter/notificationservice/api:thrift-scala", + "notificationservice/thrift/src/main/thrift/com/twitter/notificationservice/badgecount-api:thrift-scala", + "notificationservice/thrift/src/main/thrift/com/twitter/notificationservice/generic_notifications:thrift-scala", + "notifinfra/ni-lib/src/main/scala/com/twitter/ni/lib/logged_out_transform", + "observability/observability-manhattan-client/src/main/scala", + "onboarding/service/src/main/scala/com/twitter/onboarding/task/service/models/external", + "onboarding/service/thrift/src/main/thrift:thrift-scala", + "people-discovery/api/thrift/src/main/thrift:thrift-scala", + "periscope/api-proxy-thrift/thrift/src/main/thrift:thrift-scala", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", + "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", + "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", + "qig-ranker/thrift/src/main/thrift:thrift-scala", + "rux-ds/src/main/thrift/com/twitter/ruxds/jobs/user_past_aggregate:user_past_aggregate-scala", + "rux/common/src/main/scala/com/twitter/rux/common/encode", + "rux/common/thrift/src/main/thrift/rux-context:rux-context-scala", + "rux/common/thrift/src/main/thrift/strato:strato-scala", + "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", + "scrooge/scrooge-core", + "scrooge/scrooge-serializer/src/main/scala", + "sensitive-ds/src/main/thrift/com/twitter/scio/nsfw_user_segmentation:nsfw_user_segmentation-scala", + "servo/decider/src/main/scala", + "servo/request/src/main/scala", + "servo/util/src/main/scala", + "src/java/com/twitter/ml/api:api-base", + "src/java/com/twitter/ml/prediction/core", + "src/scala/com/twitter/frigate/data_pipeline/common", + "src/scala/com/twitter/frigate/data_pipeline/embedding_cg:embedding_cg-test-user-ids", + "src/scala/com/twitter/frigate/data_pipeline/features_common", + "src/scala/com/twitter/frigate/news_article_recs/news_articles_metadata:thrift-scala", + "src/scala/com/twitter/frontpage/stream/util", + "src/scala/com/twitter/language/normalization", + "src/scala/com/twitter/ml/api/embedding", + "src/scala/com/twitter/ml/api/util:datarecord", + "src/scala/com/twitter/ml/featurestore/catalog/entities/core", + "src/scala/com/twitter/ml/featurestore/catalog/entities/magicrecs", + "src/scala/com/twitter/ml/featurestore/catalog/features/core:aggregate", + "src/scala/com/twitter/ml/featurestore/catalog/features/cuad:aggregate", + "src/scala/com/twitter/ml/featurestore/catalog/features/embeddings", + "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:aggregate", + "src/scala/com/twitter/ml/featurestore/catalog/features/topic_signals:aggregate", + "src/scala/com/twitter/ml/featurestore/lib", + "src/scala/com/twitter/ml/featurestore/lib/data", + "src/scala/com/twitter/ml/featurestore/lib/dynamic", + "src/scala/com/twitter/ml/featurestore/lib/entity", + "src/scala/com/twitter/ml/featurestore/lib/online", + "src/scala/com/twitter/recommendation/interests/discovery/core/config", + "src/scala/com/twitter/recommendation/interests/discovery/core/deploy", + "src/scala/com/twitter/recommendation/interests/discovery/core/model", + "src/scala/com/twitter/recommendation/interests/discovery/popgeo/deploy", + "src/scala/com/twitter/simclusters_v2/common", + "src/scala/com/twitter/storehaus_internal/manhattan", + "src/scala/com/twitter/storehaus_internal/manhattan/config", + "src/scala/com/twitter/storehaus_internal/memcache", + "src/scala/com/twitter/storehaus_internal/memcache/config", + "src/scala/com/twitter/storehaus_internal/util", + "src/scala/com/twitter/taxi/common", + "src/scala/com/twitter/taxi/config", + "src/scala/com/twitter/taxi/deploy", + "src/scala/com/twitter/taxi/trending/common", + "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", + "src/thrift/com/twitter/clientapp/gen:clientapp-scala", + "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", + "src/thrift/com/twitter/escherbird/common:constants-scala", + "src/thrift/com/twitter/escherbird/metadata:megadata-scala", + "src/thrift/com/twitter/escherbird/metadata:metadata-service-scala", + "src/thrift/com/twitter/escherbird/search:search-service-scala", + "src/thrift/com/twitter/expandodo:only-scala", + "src/thrift/com/twitter/frigate:frigate-common-thrift-scala", + "src/thrift/com/twitter/frigate:frigate-ml-thrift-scala", + "src/thrift/com/twitter/frigate:frigate-notification-thrift-scala", + "src/thrift/com/twitter/frigate:frigate-secondary-accounts-thrift-scala", + "src/thrift/com/twitter/frigate:frigate-thrift-scala", + "src/thrift/com/twitter/frigate:frigate-user-media-representation-thrift-scala", + "src/thrift/com/twitter/frigate/data_pipeline:frigate-user-history-thrift-scala", + "src/thrift/com/twitter/frigate/dau_model:frigate-dau-thrift-scala", + "src/thrift/com/twitter/frigate/magic_events:frigate-magic-events-thrift-scala", + "src/thrift/com/twitter/frigate/magic_events/scribe:thrift-scala", + "src/thrift/com/twitter/frigate/pushcap:frigate-pushcap-thrift-scala", + "src/thrift/com/twitter/frigate/pushservice:frigate-pushservice-thrift-scala", + "src/thrift/com/twitter/frigate/scribe:frigate-scribe-thrift-scala", + "src/thrift/com/twitter/frigate/subscribed_search:frigate-subscribed-search-thrift-scala", + "src/thrift/com/twitter/frigate/user_states:frigate-userstates-thrift-scala", + "src/thrift/com/twitter/geoduck:geoduck-scala", + "src/thrift/com/twitter/gizmoduck:thrift-scala", + "src/thrift/com/twitter/gizmoduck:user-thrift-scala", + "src/thrift/com/twitter/hermit:hermit-scala", + "src/thrift/com/twitter/hermit/pop_geo:hermit-pop-geo-scala", + "src/thrift/com/twitter/hermit/stp:hermit-stp-scala", + "src/thrift/com/twitter/ibis:service-scala", + "src/thrift/com/twitter/manhattan:v1-scala", + "src/thrift/com/twitter/manhattan:v2-scala", + "src/thrift/com/twitter/ml/api:data-java", + "src/thrift/com/twitter/ml/api:data-scala", + "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-scala", + "src/thrift/com/twitter/ml/featurestore/timelines:ml-features-timelines-strato", + "src/thrift/com/twitter/ml/prediction_service:prediction_service-java", + "src/thrift/com/twitter/permissions_storage:thrift-scala", + "src/thrift/com/twitter/pink-floyd/thrift:thrift-scala", + "src/thrift/com/twitter/recos:recos-common-scala", + "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", + "src/thrift/com/twitter/recos/user_user_graph:user_user_graph-scala", + "src/thrift/com/twitter/relevance/feature_store:feature_store-scala", + "src/thrift/com/twitter/search:earlybird-scala", + "src/thrift/com/twitter/search/common:features-scala", + "src/thrift/com/twitter/search/query_interaction_graph:query_interaction_graph-scala", + "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", + "src/thrift/com/twitter/service/metastore/gen:thrift-scala", + "src/thrift/com/twitter/service/scarecrow/gen:scarecrow-scala", + "src/thrift/com/twitter/service/scarecrow/gen:tiered-actions-scala", + "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", + "src/thrift/com/twitter/socialgraph:thrift-scala", + "src/thrift/com/twitter/spam/rtf:safety-level-scala", + "src/thrift/com/twitter/timelinemixer:thrift-scala", + "src/thrift/com/twitter/timelinemixer/server/internal:thrift-scala", + "src/thrift/com/twitter/timelines/author_features/user_health:thrift-scala", + "src/thrift/com/twitter/timelines/real_graph:real_graph-scala", + "src/thrift/com/twitter/timelinescorer:thrift-scala", + "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", + "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", + "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", + "src/thrift/com/twitter/trends/common:common-scala", + "src/thrift/com/twitter/trends/trip_v1:trip-tweets-thrift-scala", + "src/thrift/com/twitter/tweetypie:service-scala", + "src/thrift/com/twitter/tweetypie:tweet-scala", + "src/thrift/com/twitter/user_session_store:thrift-scala", + "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", + "src/thrift/com/twitter/wtf/interest:interest-thrift-scala", + "src/thrift/com/twitter/wtf/scalding/common:thrift-scala", + "stitch/stitch-core", + "stitch/stitch-gizmoduck", + "stitch/stitch-socialgraph/src/main/scala", + "stitch/stitch-storehaus/src/main/scala", + "stitch/stitch-tweetypie/src/main/scala", + "storage/clients/manhattan/client/src/main/scala", + "strato/config/columns/clients:clients-strato-client", + "strato/config/columns/geo/user:user-strato-client", + "strato/config/columns/globe/curation:curation-strato-client", + "strato/config/columns/interests:interests-strato-client", + "strato/config/columns/ml/featureStore:featureStore-strato-client", + "strato/config/columns/notifications:notifications-strato-client", + "strato/config/columns/notifinfra:notifinfra-strato-client", + "strato/config/columns/periscope:periscope-strato-client", + "strato/config/columns/rux", + "strato/config/columns/rux:rux-strato-client", + "strato/config/columns/rux/open-app:open-app-strato-client", + "strato/config/columns/socialgraph/graphs:graphs-strato-client", + "strato/config/columns/socialgraph/service/soft_users:soft_users-strato-client", + "strato/config/columns/translation/service:service-strato-client", + "strato/config/columns/translation/service/platform:platform-strato-client", + "strato/config/columns/trends/trip:trip-strato-client", + "strato/config/src/thrift/com/twitter/strato/columns/frigate:logged-out-web-notifications-scala", + "strato/config/src/thrift/com/twitter/strato/columns/notifications:thrift-scala", + "strato/src/main/scala/com/twitter/strato/config", + "strato/src/main/scala/com/twitter/strato/response", + "thrift-web-forms", + "timeline-training-service/service/thrift/src/main/thrift:thrift-scala", + "timelines/src/main/scala/com/twitter/timelines/features/app", + "topic-social-proof/server/src/main/thrift:thrift-scala", + "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", + "topiclisting/topiclisting-utt/src/main/scala/com/twitter/topiclisting/utt", + "trends/common/src/main/thrift/com/twitter/trends/common:thrift-scala", + "tweetypie/src/scala/com/twitter/tweetypie/tweettext", + "twitter-context/src/main/scala", + "twitter-server-internal", + "twitter-server/server/src/main/scala", + "twitter-text/lib/java/src/main/java/com/twitter/twittertext", + "twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine:prediction_engine_mkl", + "ubs/common/src/main/thrift/com/twitter/ubs:broadcast-thrift-scala", + "ubs/common/src/main/thrift/com/twitter/ubs:seller_application-thrift-scala", + "user_session_store/src/main/scala/com/twitter/user_session_store/impl/manhattan/readwrite", + "util-internal/scribe", + "util-internal/tunable/src/main/scala/com/twitter/util/tunable", + "util/util-app", + "util/util-hashing/src/main/scala", + "util/util-slf4j-api/src/main/scala", + "util/util-stats/src/main/scala", + "visibility/lib/src/main/scala/com/twitter/visibility/builder", + "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/push_service", + "visibility/lib/src/main/scala/com/twitter/visibility/interfaces/spaces", + "visibility/lib/src/main/scala/com/twitter/visibility/util", + ], + exports = [ + "strato/config/src/thrift/com/twitter/strato/columns/frigate:logged-out-web-notifications-scala", + ], +) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.scala new file mode 100644 index 0000000000..b13d3b0937 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushMixerThriftServerWarmupHandler.scala @@ -0,0 +1,93 @@ +package com.twitter.frigate.pushservice + +import com.google.inject.Inject +import com.google.inject.Singleton +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.thrift.ClientId +import com.twitter.finatra.thrift.routing.ThriftWarmup +import com.twitter.util.logging.Logging +import com.twitter.inject.utils.Handler +import com.twitter.frigate.pushservice.{thriftscala => t} +import com.twitter.frigate.thriftscala.NotificationDisplayLocation +import com.twitter.util.Stopwatch +import com.twitter.scrooge.Request +import com.twitter.scrooge.Response +import com.twitter.util.Return +import com.twitter.util.Throw +import com.twitter.util.Try + +/** + * Warms up the refresh request path. + * If service is running as pushservice-send then the warmup does nothing. + * + * When making the warmup refresh requests we + * - Set skipFilters to true to execute as much of the request path as possible + * - Set darkWrite to true to prevent sending a push + */ +@Singleton +class PushMixerThriftServerWarmupHandler @Inject() ( + warmup: ThriftWarmup, + serviceIdentifier: ServiceIdentifier) + extends Handler + with Logging { + + private val clientId = ClientId("thrift-warmup-client") + + def handle(): Unit = { + val refreshServices = Set( + "frigate-pushservice", + "frigate-pushservice-canary", + "frigate-pushservice-canary-control", + "frigate-pushservice-canary-treatment" + ) + val isRefresh = refreshServices.contains(serviceIdentifier.service) + if (isRefresh && !serviceIdentifier.isLocal) refreshWarmup() + } + + def refreshWarmup(): Unit = { + val elapsed = Stopwatch.start() + val testIds = Seq( + 1, + 2, + 3 + ) + try { + clientId.asCurrent { + testIds.foreach { id => + val warmupReq = warmupQuery(id) + info(s"Sending warm-up request to service with query: $warmupReq") + warmup.sendRequest( + method = t.PushService.Refresh, + req = Request(t.PushService.Refresh.Args(warmupReq)))(assertWarmupResponse) + } + } + } catch { + case e: Throwable => + error(e.getMessage, e) + } + info(s"Warm up complete. Time taken: ${elapsed().toString}") + } + + private def warmupQuery(userId: Long): t.RefreshRequest = { + t.RefreshRequest( + userId = userId, + notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, + context = Some( + t.PushContext( + skipFilters = Some(true), + darkWrite = Some(true) + )) + ) + } + + private def assertWarmupResponse( + result: Try[Response[t.PushService.Refresh.SuccessType]] + ): Unit = { + result match { + case Return(_) => // ok + case Throw(exception) => + warn("Error performing warm-up request.") + error(exception.getMessage, exception) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.scala new file mode 100644 index 0000000000..c60f6e3522 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/PushServiceMain.scala @@ -0,0 +1,193 @@ +package com.twitter.frigate.pushservice + +import com.twitter.discovery.common.environment.modules.EnvironmentModule +import com.twitter.finagle.Filter +import com.twitter.finatra.annotations.DarkTrafficFilterType +import com.twitter.finatra.decider.modules.DeciderModule +import com.twitter.finatra.http.HttpServer +import com.twitter.finatra.http.filters.CommonFilters +import com.twitter.finatra.http.routing.HttpRouter +import com.twitter.finatra.mtls.http.{Mtls => HttpMtls} +import com.twitter.finatra.mtls.thriftmux.{Mtls => ThriftMtls} +import com.twitter.finatra.mtls.thriftmux.filters.MtlsServerSessionTrackerFilter +import com.twitter.finatra.thrift.ThriftServer +import com.twitter.finatra.thrift.filters.ExceptionMappingFilter +import com.twitter.finatra.thrift.filters.LoggingMDCFilter +import com.twitter.finatra.thrift.filters.StatsFilter +import com.twitter.finatra.thrift.filters.ThriftMDCFilter +import com.twitter.finatra.thrift.filters.TraceIdMDCFilter +import com.twitter.finatra.thrift.routing.ThriftRouter +import com.twitter.frigate.common.logger.MRLoggerGlobalVariables +import com.twitter.frigate.pushservice.controller.PushServiceController +import com.twitter.frigate.pushservice.module._ +import com.twitter.inject.TwitterModule +import com.twitter.inject.annotations.Flags +import com.twitter.inject.thrift.modules.ThriftClientIdModule +import com.twitter.logging.BareFormatter +import com.twitter.logging.Level +import com.twitter.logging.LoggerFactory +import com.twitter.logging.{Logging => JLogging} +import com.twitter.logging.QueueingHandler +import com.twitter.logging.ScribeHandler +import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule +import com.twitter.product_mixer.core.module.ABDeciderModule +import com.twitter.product_mixer.core.module.FeatureSwitchesModule +import com.twitter.product_mixer.core.module.StratoClientModule + +object PushServiceMain extends PushServiceFinatraServer + +class PushServiceFinatraServer + extends ThriftServer + with ThriftMtls + with HttpServer + with HttpMtls + with JLogging { + + override val name = "PushService" + + override val modules: Seq[TwitterModule] = { + Seq( + ABDeciderModule, + DeciderModule, + FeatureSwitchesModule, + FilterModule, + FlagModule, + EnvironmentModule, + ThriftClientIdModule, + DeployConfigModule, + ProductMixerFlagModule, + StratoClientModule, + PushHandlerModule, + PushTargetUserBuilderModule, + PushServiceDarkTrafficModule, + LoggedOutPushTargetUserBuilderModule, + new ThriftWebFormsModule(this), + ) + } + + override def configureThrift(router: ThriftRouter): Unit = { + router + .filter[ExceptionMappingFilter] + .filter[LoggingMDCFilter] + .filter[TraceIdMDCFilter] + .filter[ThriftMDCFilter] + .filter[MtlsServerSessionTrackerFilter] + .filter[StatsFilter] + .filter[Filter.TypeAgnostic, DarkTrafficFilterType] + .add[PushServiceController] + } + + override def configureHttp(router: HttpRouter): Unit = + router + .filter[CommonFilters] + + override protected def start(): Unit = { + MRLoggerGlobalVariables.setRequiredFlags( + traceLogFlag = injector.instance[Boolean](Flags.named(FlagModule.mrLoggerIsTraceAll.name)), + nthLogFlag = injector.instance[Boolean](Flags.named(FlagModule.mrLoggerNthLog.name)), + nthLogValFlag = injector.instance[Long](Flags.named(FlagModule.mrLoggerNthVal.name)) + ) + } + + override protected def warmup(): Unit = { + handle[PushMixerThriftServerWarmupHandler]() + } + + override protected def configureLoggerFactories(): Unit = { + loggerFactories.foreach { _() } + } + + override def loggerFactories: List[LoggerFactory] = { + val scribeScope = statsReceiver.scope("scribe") + List( + LoggerFactory( + level = Some(levelFlag()), + handlers = handlers + ), + LoggerFactory( + node = "request_scribe", + level = Some(Level.INFO), + useParents = false, + handlers = QueueingHandler( + maxQueueSize = 10000, + handler = ScribeHandler( + category = "frigate_pushservice_log", + formatter = BareFormatter, + statsReceiver = scribeScope.scope("frigate_pushservice_log") + ) + ) :: Nil + ), + LoggerFactory( + node = "notification_scribe", + level = Some(Level.INFO), + useParents = false, + handlers = QueueingHandler( + maxQueueSize = 10000, + handler = ScribeHandler( + category = "frigate_notifier", + formatter = BareFormatter, + statsReceiver = scribeScope.scope("frigate_notifier") + ) + ) :: Nil + ), + LoggerFactory( + node = "push_scribe", + level = Some(Level.INFO), + useParents = false, + handlers = QueueingHandler( + maxQueueSize = 10000, + handler = ScribeHandler( + category = "test_frigate_push", + formatter = BareFormatter, + statsReceiver = scribeScope.scope("test_frigate_push") + ) + ) :: Nil + ), + LoggerFactory( + node = "push_subsample_scribe", + level = Some(Level.INFO), + useParents = false, + handlers = QueueingHandler( + maxQueueSize = 2500, + handler = ScribeHandler( + category = "magicrecs_candidates_subsample_scribe", + maxMessagesPerTransaction = 250, + maxMessagesToBuffer = 2500, + formatter = BareFormatter, + statsReceiver = scribeScope.scope("magicrecs_candidates_subsample_scribe") + ) + ) :: Nil + ), + LoggerFactory( + node = "mr_request_scribe", + level = Some(Level.INFO), + useParents = false, + handlers = QueueingHandler( + maxQueueSize = 2500, + handler = ScribeHandler( + category = "mr_request_scribe", + maxMessagesPerTransaction = 250, + maxMessagesToBuffer = 2500, + formatter = BareFormatter, + statsReceiver = scribeScope.scope("mr_request_scribe") + ) + ) :: Nil + ), + LoggerFactory( + node = "high_quality_candidates_scribe", + level = Some(Level.INFO), + useParents = false, + handlers = QueueingHandler( + maxQueueSize = 2500, + handler = ScribeHandler( + category = "frigate_high_quality_candidates_log", + maxMessagesPerTransaction = 250, + maxMessagesToBuffer = 2500, + formatter = BareFormatter, + statsReceiver = scribeScope.scope("high_quality_candidates_scribe") + ) + ) :: Nil + ), + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.scala new file mode 100644 index 0000000000..946923fb9a --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ContentRecommenderMixerAdaptor.scala @@ -0,0 +1,323 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.contentrecommender.thriftscala.MetricTag +import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest +import com.twitter.cr_mixer.thriftscala.NotificationsContext +import com.twitter.cr_mixer.thriftscala.Product +import com.twitter.cr_mixer.thriftscala.ProductContext +import com.twitter.cr_mixer.thriftscala.{MetricTag => CrMixerMetricTag} +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.AlgorithmScore +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.CandidateSourceEligible +import com.twitter.frigate.common.base.CrMixerCandidate +import com.twitter.frigate.common.base.TopicCandidate +import com.twitter.frigate.common.base.TopicProofTweetCandidate +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutInNetworkTweets +import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.store.CrMixerTweetStore +import com.twitter.frigate.pushservice.store.UttEntityHydrationStore +import com.twitter.frigate.pushservice.util.AdaptorUtils +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.pushservice.util.TopicsUtil +import com.twitter.frigate.pushservice.util.TweetWithTopicProof +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.product_mixer.core.thriftscala.ClientContext +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.topiclisting.utt.LocalizedEntity +import com.twitter.tsp.thriftscala.TopicSocialProofRequest +import com.twitter.tsp.thriftscala.TopicSocialProofResponse +import com.twitter.util.Future +import scala.collection.Map + +case class ContentRecommenderMixerAdaptor( + crMixerTweetStore: CrMixerTweetStore, + tweetyPieStore: ReadableStore[Long, TweetyPieResult], + edgeStore: ReadableStore[RelationEdge, Boolean], + topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], + uttEntityHydrationStore: UttEntityHydrationStore, + globalStats: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + + override val name: String = this.getClass.getSimpleName + + private[this] val stats = globalStats.scope("ContentRecommenderMixerAdaptor") + private[this] val numOfValidAuthors = stats.stat("num_of_valid_authors") + private[this] val numOutOfMaximumDropped = stats.stat("dropped_due_out_of_maximum") + private[this] val totalInputRecs = stats.counter("input_recs") + private[this] val totalOutputRecs = stats.stat("output_recs") + private[this] val totalRequests = stats.counter("total_requests") + private[this] val nonReplyTweetsCounter = stats.counter("non_reply_tweets") + private[this] val totalOutNetworkRecs = stats.counter("out_network_tweets") + private[this] val totalInNetworkRecs = stats.counter("in_network_tweets") + + /** + * Builds OON raw candidates based on input OON Tweets + */ + def buildOONRawCandidates( + inputTarget: Target, + oonTweets: Seq[TweetyPieResult], + tweetScoreMap: Map[Long, Double], + tweetIdToTagsMap: Map[Long, Seq[CrMixerMetricTag]], + maxNumOfCandidates: Int + ): Option[Seq[RawCandidate]] = { + val cands = oonTweets.flatMap { tweetResult => + val tweetId = tweetResult.tweet.id + generateOONRawCandidate( + inputTarget, + tweetId, + Some(tweetResult), + tweetScoreMap, + tweetIdToTagsMap + ) + } + + val candidates = restrict( + maxNumOfCandidates, + cands, + numOutOfMaximumDropped, + totalOutputRecs + ) + + Some(candidates) + } + + /** + * Builds a single RawCandidate With TopicProofTweetCandidate + */ + def buildTopicTweetRawCandidate( + inputTarget: Target, + tweetWithTopicProof: TweetWithTopicProof, + localizedEntity: LocalizedEntity, + tags: Option[Seq[MetricTag]], + ): RawCandidate with TopicProofTweetCandidate = { + new RawCandidate with TopicProofTweetCandidate { + override def target: Target = inputTarget + override def topicListingSetting: Option[String] = Some( + tweetWithTopicProof.topicListingSetting) + override def tweetId: Long = tweetWithTopicProof.tweetId + override def tweetyPieResult: Option[TweetyPieResult] = Some( + tweetWithTopicProof.tweetyPieResult) + override def semanticCoreEntityId: Option[Long] = Some(tweetWithTopicProof.topicId) + override def localizedUttEntity: Option[LocalizedEntity] = Some(localizedEntity) + override def algorithmCR: Option[String] = tweetWithTopicProof.algorithmCR + override def tagsCR: Option[Seq[MetricTag]] = tags + override def isOutOfNetwork: Boolean = tweetWithTopicProof.isOON + } + } + + /** + * Takes a group of TopicTweets and transforms them into RawCandidates + */ + def buildTopicTweetRawCandidates( + inputTarget: Target, + topicProofCandidates: Seq[TweetWithTopicProof], + tweetIdToTagsMap: Map[Long, Seq[CrMixerMetricTag]], + maxNumberOfCands: Int + ): Future[Option[Seq[RawCandidate]]] = { + val semanticCoreEntityIds = topicProofCandidates + .map(_.topicId) + .toSet + + TopicsUtil + .getLocalizedEntityMap(inputTarget, semanticCoreEntityIds, uttEntityHydrationStore) + .map { localizedEntityMap => + val rawCandidates = topicProofCandidates.collect { + case topicSocialProof: TweetWithTopicProof + if localizedEntityMap.contains(topicSocialProof.topicId) => + // Once we deprecate CR calls, we should replace this code to use the CrMixerMetricTag + val tags = tweetIdToTagsMap.get(topicSocialProof.tweetId).map { + _.flatMap { tag => MetricTag.get(tag.value) } + } + buildTopicTweetRawCandidate( + inputTarget, + topicSocialProof, + localizedEntityMap(topicSocialProof.topicId), + tags + ) + } + + val candResult = restrict( + maxNumberOfCands, + rawCandidates, + numOutOfMaximumDropped, + totalOutputRecs + ) + + Some(candResult) + } + } + + private def generateOONRawCandidate( + inputTarget: Target, + id: Long, + result: Option[TweetyPieResult], + tweetScoreMap: Map[Long, Double], + tweetIdToTagsMap: Map[Long, Seq[CrMixerMetricTag]] + ): Option[RawCandidate with TweetCandidate] = { + val tagsFromCR = tweetIdToTagsMap.get(id).map { _.flatMap { tag => MetricTag.get(tag.value) } } + val candidate = new RawCandidate with CrMixerCandidate with TopicCandidate with AlgorithmScore { + override val tweetId = id + override val target = inputTarget + override val tweetyPieResult = result + override val localizedUttEntity = None + override val semanticCoreEntityId = None + override def commonRecType = + getMediaBasedCRT( + CommonRecommendationType.TwistlyTweet, + CommonRecommendationType.TwistlyPhoto, + CommonRecommendationType.TwistlyVideo) + override def tagsCR = tagsFromCR + override def algorithmScore = tweetScoreMap.get(id) + override def algorithmCR = None + } + Some(candidate) + } + + private def restrict( + maxNumToReturn: Int, + candidates: Seq[RawCandidate], + numOutOfMaximumDropped: Stat, + totalOutputRecs: Stat + ): Seq[RawCandidate] = { + val newCandidates = candidates.take(maxNumToReturn) + val numDropped = candidates.length - newCandidates.length + numOutOfMaximumDropped.add(numDropped) + totalOutputRecs.add(newCandidates.size) + newCandidates + } + + private def buildCrMixerRequest( + target: Target, + countryCode: Option[String], + language: Option[String], + seenTweets: Seq[Long] + ): CrMixerTweetRequest = { + CrMixerTweetRequest( + clientContext = ClientContext( + userId = Some(target.targetId), + countryCode = countryCode, + languageCode = language + ), + product = Product.Notifications, + productContext = Some(ProductContext.NotificationsContext(NotificationsContext())), + excludedTweetIds = Some(seenTweets) + ) + } + + private def selectCandidatesToSendBasedOnSettings( + isRecommendationsEligible: Boolean, + isTopicsEligible: Boolean, + oonRawCandidates: Option[Seq[RawCandidate]], + topicTweetCandidates: Option[Seq[RawCandidate]] + ): Option[Seq[RawCandidate]] = { + if (isRecommendationsEligible && isTopicsEligible) { + Some(topicTweetCandidates.getOrElse(Seq.empty) ++ oonRawCandidates.getOrElse(Seq.empty)) + } else if (isRecommendationsEligible) { + oonRawCandidates + } else if (isTopicsEligible) { + topicTweetCandidates + } else None + } + + override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { + Future + .join( + target.seenTweetIds, + target.countryCode, + target.inferredUserDeviceLanguage, + PushDeviceUtil.isTopicsEligible(target), + PushDeviceUtil.isRecommendationsEligible(target) + ).flatMap { + case (seenTweets, countryCode, language, isTopicsEligible, isRecommendationsEligible) => + val request = buildCrMixerRequest(target, countryCode, language, seenTweets) + crMixerTweetStore.getTweetRecommendations(request).flatMap { + case Some(response) => + totalInputRecs.incr(response.tweets.size) + totalRequests.incr() + AdaptorUtils + .getTweetyPieResults( + response.tweets.map(_.tweetId).toSet, + tweetyPieStore).flatMap { tweetyPieResultMap => + filterOutInNetworkTweets( + target, + filterOutReplyTweet(tweetyPieResultMap.toMap, nonReplyTweetsCounter), + edgeStore, + numOfValidAuthors).flatMap { + outNetworkTweetsWithId: Seq[(Long, TweetyPieResult)] => + totalOutNetworkRecs.incr(outNetworkTweetsWithId.size) + totalInNetworkRecs.incr(response.tweets.size - outNetworkTweetsWithId.size) + val outNetworkTweets: Seq[TweetyPieResult] = outNetworkTweetsWithId.map { + case (_, tweetyPieResult) => tweetyPieResult + } + + val tweetIdToTagsMap = response.tweets.map { tweet => + tweet.tweetId -> tweet.metricTags.getOrElse(Seq.empty) + }.toMap + + val tweetScoreMap = response.tweets.map { tweet => + tweet.tweetId -> tweet.score + }.toMap + + val maxNumOfCandidates = + target.params(PushFeatureSwitchParams.NumberOfMaxCrMixerCandidatesParam) + + val oonRawCandidates = + buildOONRawCandidates( + target, + outNetworkTweets, + tweetScoreMap, + tweetIdToTagsMap, + maxNumOfCandidates) + + TopicsUtil + .getTopicSocialProofs( + target, + outNetworkTweets, + topicSocialProofServiceStore, + edgeStore, + PushFeatureSwitchParams.TopicProofTweetCandidatesTopicScoreThreshold).flatMap { + tweetsWithTopicProof => + buildTopicTweetRawCandidates( + target, + tweetsWithTopicProof, + tweetIdToTagsMap, + maxNumOfCandidates) + }.map { topicTweetCandidates => + selectCandidatesToSendBasedOnSettings( + isRecommendationsEligible, + isTopicsEligible, + oonRawCandidates, + topicTweetCandidates) + } + } + } + case _ => Future.None + } + } + } + + /** + * For a user to be available the following news to happen + */ + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + Future + .join( + PushDeviceUtil.isRecommendationsEligible(target), + PushDeviceUtil.isTopicsEligible(target) + ).map { + case (isRecommendationsEligible, isTopicsEligible) => + (isRecommendationsEligible || isTopicsEligible) && + target.params(PushParams.ContentRecommenderMixerAdaptorDecider) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.scala new file mode 100644 index 0000000000..ab631841a7 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/EarlyBirdFirstDegreeCandidateAdaptor.scala @@ -0,0 +1,293 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.candidate._ +import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.hermit.store.tweetypie.UserTweet +import com.twitter.recos.recos_common.thriftscala.SocialProofType +import com.twitter.search.common.features.thriftscala.ThriftSearchResultFeatures +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.configapi.Param +import com.twitter.util.Future +import com.twitter.util.Time +import scala.collection.Map + +case class EarlyBirdFirstDegreeCandidateAdaptor( + earlyBirdFirstDegreeCandidates: CandidateSource[ + EarlybirdCandidateSource.Query, + EarlybirdCandidate + ], + tweetyPieStore: ReadableStore[Long, TweetyPieResult], + tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], + userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], + maxResultsParam: Param[Int], + globalStats: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + + type EBCandidate = EarlybirdCandidate with TweetDetails + private val stats = globalStats.scope("EarlyBirdFirstDegreeAdaptor") + private val earlyBirdCandsStat: Stat = stats.stat("early_bird_cands_dist") + private val emptyEarlyBirdCands = stats.counter("empty_early_bird_candidates") + private val seedSetEmpty = stats.counter("empty_seedset") + private val seenTweetsStat = stats.stat("filtered_by_seen_tweets") + private val emptyTweetyPieResult = stats.stat("empty_tweetypie_result") + private val nonReplyTweetsCounter = stats.counter("non_reply_tweets") + private val enableRetweets = stats.counter("enable_retweets") + private val f1withoutSocialContexts = stats.counter("f1_without_social_context") + private val userTweetTweetyPieStoreCounter = stats.counter("user_tweet_tweetypie_store") + + override val name: String = earlyBirdFirstDegreeCandidates.name + + private def getAllSocialContextActions( + socialProofTypes: Seq[(SocialProofType, Seq[Long])] + ): Seq[SocialContextAction] = { + socialProofTypes.flatMap { + case (SocialProofType.Favorite, scIds) => + scIds.map { scId => + SocialContextAction( + scId, + Time.now.inMilliseconds, + socialContextActionType = Some(SocialContextActionType.Favorite) + ) + } + case (SocialProofType.Retweet, scIds) => + scIds.map { scId => + SocialContextAction( + scId, + Time.now.inMilliseconds, + socialContextActionType = Some(SocialContextActionType.Retweet) + ) + } + case (SocialProofType.Reply, scIds) => + scIds.map { scId => + SocialContextAction( + scId, + Time.now.inMilliseconds, + socialContextActionType = Some(SocialContextActionType.Reply) + ) + } + case (SocialProofType.Tweet, scIds) => + scIds.map { scId => + SocialContextAction( + scId, + Time.now.inMilliseconds, + socialContextActionType = Some(SocialContextActionType.Tweet) + ) + } + case _ => Nil + } + } + + private def generateRetweetCandidate( + inputTarget: Target, + candidate: EBCandidate, + scIds: Seq[Long], + socialProofTypes: Seq[(SocialProofType, Seq[Long])] + ): RawCandidate = { + val scActions = scIds.map { scId => SocialContextAction(scId, Time.now.inMilliseconds) } + new RawCandidate with TweetRetweetCandidate with EarlybirdTweetFeatures { + override val socialContextActions = scActions + override val socialContextAllTypeActions = getAllSocialContextActions(socialProofTypes) + override val tweetId = candidate.tweetId + override val target = inputTarget + override val tweetyPieResult = candidate.tweetyPieResult + override val features = candidate.features + } + } + + private def generateF1CandidateWithoutSocialContext( + inputTarget: Target, + candidate: EBCandidate + ): RawCandidate = { + f1withoutSocialContexts.incr() + new RawCandidate with F1FirstDegree with EarlybirdTweetFeatures { + override val tweetId = candidate.tweetId + override val target = inputTarget + override val tweetyPieResult = candidate.tweetyPieResult + override val features = candidate.features + } + } + + private def generateEarlyBirdCandidate( + id: Long, + result: Option[TweetyPieResult], + ebFeatures: Option[ThriftSearchResultFeatures] + ): EBCandidate = { + new EarlybirdCandidate with TweetDetails { + override val tweetyPieResult: Option[TweetyPieResult] = result + override val tweetId: Long = id + override val features: Option[ThriftSearchResultFeatures] = ebFeatures + } + } + + private def filterOutSeenTweets(seenTweetIds: Seq[Long], inputTweetIds: Seq[Long]): Seq[Long] = { + inputTweetIds.filterNot(seenTweetIds.contains) + } + + private def filterInvalidTweets( + tweetIds: Seq[Long], + target: Target + ): Future[Seq[(Long, TweetyPieResult)]] = { + + val resMap = { + if (target.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors)) { + userTweetTweetyPieStoreCounter.incr() + val keys = tweetIds.map { tweetId => + UserTweet(tweetId, Some(target.targetId)) + } + + userTweetTweetyPieStore + .multiGet(keys.toSet).map { + case (userTweet, resultFut) => + userTweet.tweetId -> resultFut + }.toMap + } else { + (target.params(PushFeatureSwitchParams.EnableVFInTweetypie) match { + case true => tweetyPieStore + case false => tweetyPieStoreNoVF + }).multiGet(tweetIds.toSet) + } + } + Future.collect(resMap).map { tweetyPieResultMap => + val cands = filterOutReplyTweet(tweetyPieResultMap, nonReplyTweetsCounter).collect { + case (id: Long, Some(result)) => + id -> result + } + + emptyTweetyPieResult.add(tweetyPieResultMap.size - cands.size) + cands.toSeq + } + } + + private def getEBRetweetCandidates( + inputTarget: Target, + retweets: Seq[(Long, TweetyPieResult)] + ): Seq[RawCandidate] = { + retweets.flatMap { + case (_, tweetypieResult) => + tweetypieResult.tweet.coreData.flatMap { coreData => + tweetypieResult.sourceTweet.map { sourceTweet => + val tweetId = sourceTweet.id + val scId = coreData.userId + val socialProofTypes = Seq((SocialProofType.Retweet, Seq(scId))) + val candidate = generateEarlyBirdCandidate( + tweetId, + Some(TweetyPieResult(sourceTweet, None, None)), + None + ) + generateRetweetCandidate( + inputTarget, + candidate, + Seq(scId), + socialProofTypes + ) + } + } + } + } + + private def getEBFirstDegreeCands( + tweets: Seq[(Long, TweetyPieResult)], + ebTweetIdMap: Map[Long, Option[ThriftSearchResultFeatures]] + ): Seq[EBCandidate] = { + tweets.map { + case (id, tweetypieResult) => + val features = ebTweetIdMap.getOrElse(id, None) + generateEarlyBirdCandidate(id, Some(tweetypieResult), features) + } + } + + /** + * Returns a combination of raw candidates made of: f1 recs, topic social proof recs, sc recs and retweet candidates + */ + def buildRawCandidates( + inputTarget: Target, + firstDegreeCandidates: Seq[EBCandidate], + retweetCandidates: Seq[RawCandidate] + ): Seq[RawCandidate] = { + val hydratedF1Recs = + firstDegreeCandidates.map(generateF1CandidateWithoutSocialContext(inputTarget, _)) + hydratedF1Recs ++ retweetCandidates + } + + override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { + inputTarget.seedsWithWeight.flatMap { seedsetOpt => + val seedsetMap = seedsetOpt.getOrElse(Map.empty) + + if (seedsetMap.isEmpty) { + seedSetEmpty.incr() + Future.None + } else { + val maxResultsToReturn = inputTarget.params(maxResultsParam) + val maxTweetAge = inputTarget.params(PushFeatureSwitchParams.F1CandidateMaxTweetAgeParam) + val earlybirdQuery = EarlybirdCandidateSource.Query( + maxNumResultsToReturn = maxResultsToReturn, + seedset = seedsetMap, + maxConsecutiveResultsByTheSameUser = Some(1), + maxTweetAge = maxTweetAge, + disableTimelinesMLModel = false, + searcherId = Some(inputTarget.targetId), + isProtectTweetsEnabled = + inputTarget.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors), + followedUserIds = Some(seedsetMap.keySet.toSeq) + ) + + Future + .join(inputTarget.seenTweetIds, earlyBirdFirstDegreeCandidates.get(earlybirdQuery)) + .flatMap { + case (seenTweetIds, Some(candidates)) => + earlyBirdCandsStat.add(candidates.size) + + val ebTweetIdMap = candidates.map { cand => cand.tweetId -> cand.features }.toMap + + val ebTweetIds = ebTweetIdMap.keys.toSeq + + val tweetIds = filterOutSeenTweets(seenTweetIds, ebTweetIds) + seenTweetsStat.add(ebTweetIds.size - tweetIds.size) + + filterInvalidTweets(tweetIds, inputTarget) + .map { validTweets => + val (retweets, tweets) = validTweets.partition { + case (_, tweetypieResult) => + tweetypieResult.sourceTweet.isDefined + } + + val firstDegreeCandidates = getEBFirstDegreeCands(tweets, ebTweetIdMap) + + val retweetCandidates = { + if (inputTarget.params(PushParams.EarlyBirdSCBasedCandidatesParam) && + inputTarget.params(PushParams.MRTweetRetweetRecsParam)) { + enableRetweets.incr() + getEBRetweetCandidates(inputTarget, retweets) + } else Nil + } + + Some( + buildRawCandidates( + inputTarget, + firstDegreeCandidates, + retweetCandidates + )) + } + + case _ => + emptyEarlyBirdCands.incr() + Future.None + } + } + } + } + + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + PushDeviceUtil.isRecommendationsEligible(target) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.scala new file mode 100644 index 0000000000..345fdbd3c6 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ExploreVideoTweetCandidateAdaptor.scala @@ -0,0 +1,120 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.explore_ranker.thriftscala.ExploreRankerProductResponse +import com.twitter.explore_ranker.thriftscala.ExploreRankerRequest +import com.twitter.explore_ranker.thriftscala.ExploreRankerResponse +import com.twitter.explore_ranker.thriftscala.ExploreRecommendation +import com.twitter.explore_ranker.thriftscala.ImmersiveRecsResponse +import com.twitter.explore_ranker.thriftscala.ImmersiveRecsResult +import com.twitter.explore_ranker.thriftscala.NotificationsVideoRecs +import com.twitter.explore_ranker.thriftscala.Product +import com.twitter.explore_ranker.thriftscala.ProductContext +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.CandidateSourceEligible +import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.util.AdaptorUtils +import com.twitter.frigate.pushservice.util.MediaCRT +import com.twitter.frigate.pushservice.util.PushAdaptorUtil +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.product_mixer.core.thriftscala.ClientContext +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +case class ExploreVideoTweetCandidateAdaptor( + exploreRankerStore: ReadableStore[ExploreRankerRequest, ExploreRankerResponse], + tweetyPieStore: ReadableStore[Long, TweetyPieResult], + globalStats: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + + override def name: String = this.getClass.getSimpleName + private[this] val stats = globalStats.scope("ExploreVideoTweetCandidateAdaptor") + private[this] val totalInputRecs = stats.stat("input_recs") + private[this] val totalRequests = stats.counter("total_requests") + private[this] val totalEmptyResponse = stats.counter("total_empty_response") + + private def buildExploreRankerRequest( + target: Target, + countryCode: Option[String], + language: Option[String], + ): ExploreRankerRequest = { + ExploreRankerRequest( + clientContext = ClientContext( + userId = Some(target.targetId), + countryCode = countryCode, + languageCode = language, + ), + product = Product.NotificationsVideoRecs, + productContext = Some(ProductContext.NotificationsVideoRecs(NotificationsVideoRecs())), + maxResults = Some(target.params(PushFeatureSwitchParams.MaxExploreVideoTweets)) + ) + } + + override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { + Future + .join( + target.countryCode, + target.inferredUserDeviceLanguage + ).flatMap { + case (countryCode, language) => + val request = buildExploreRankerRequest(target, countryCode, language) + exploreRankerStore.get(request).flatMap { + case Some(response) => + val exploreResonseTweetIds = response match { + case ExploreRankerResponse(ExploreRankerProductResponse + .ImmersiveRecsResponse(ImmersiveRecsResponse(immersiveRecsResult))) => + immersiveRecsResult.collect { + case ImmersiveRecsResult(ExploreRecommendation + .ExploreTweetRecommendation(exploreTweetRecommendation)) => + exploreTweetRecommendation.tweetId + } + case _ => + Seq.empty + } + + totalInputRecs.add(exploreResonseTweetIds.size) + totalRequests.incr() + AdaptorUtils + .getTweetyPieResults(exploreResonseTweetIds.toSet, tweetyPieStore).map { + tweetyPieResultMap => + val candidates = tweetyPieResultMap.values.flatten + .map(buildVideoRawCandidates(target, _)) + Some(candidates.toSeq) + } + case _ => + totalEmptyResponse.incr() + Future.None + } + case _ => + Future.None + } + } + + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + PushDeviceUtil.isRecommendationsEligible(target).map { userRecommendationsEligible => + userRecommendationsEligible && target.params(PushFeatureSwitchParams.EnableExploreVideoTweets) + } + } + private def buildVideoRawCandidates( + target: Target, + tweetyPieResult: TweetyPieResult + ): RawCandidate with OutOfNetworkTweetCandidate = { + PushAdaptorUtil.generateOutOfNetworkTweetCandidates( + inputTarget = target, + id = tweetyPieResult.tweet.id, + mediaCRT = MediaCRT( + CommonRecommendationType.ExploreVideoTweet, + CommonRecommendationType.ExploreVideoTweet, + CommonRecommendationType.ExploreVideoTweet + ), + result = Some(tweetyPieResult), + localizedEntity = None + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.scala new file mode 100644 index 0000000000..49610c6458 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/FRSTweetCandidateAdaptor.scala @@ -0,0 +1,272 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.cr_mixer.thriftscala.FrsTweetRequest +import com.twitter.cr_mixer.thriftscala.NotificationsContext +import com.twitter.cr_mixer.thriftscala.Product +import com.twitter.cr_mixer.thriftscala.ProductContext +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.CandidateSourceEligible +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.store.CrMixerTweetStore +import com.twitter.frigate.pushservice.store.UttEntityHydrationStore +import com.twitter.frigate.pushservice.util.MediaCRT +import com.twitter.frigate.pushservice.util.PushAdaptorUtil +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.pushservice.util.TopicsUtil +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.hermit.constants.AlgorithmFeedbackTokens +import com.twitter.hermit.model.Algorithm.Algorithm +import com.twitter.hermit.model.Algorithm.CrowdSearchAccounts +import com.twitter.hermit.model.Algorithm.ForwardEmailBook +import com.twitter.hermit.model.Algorithm.ForwardPhoneBook +import com.twitter.hermit.model.Algorithm.ReverseEmailBookIbis +import com.twitter.hermit.model.Algorithm.ReversePhoneBook +import com.twitter.hermit.store.tweetypie.UserTweet +import com.twitter.product_mixer.core.thriftscala.ClientContext +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.tsp.thriftscala.TopicSocialProofRequest +import com.twitter.tsp.thriftscala.TopicSocialProofResponse +import com.twitter.util.Future + +object FRSAlgorithmFeedbackTokenUtil { + private val crtsByAlgoToken = Map( + getAlgorithmToken(ReverseEmailBookIbis) -> CommonRecommendationType.ReverseAddressbookTweet, + getAlgorithmToken(ReversePhoneBook) -> CommonRecommendationType.ReverseAddressbookTweet, + getAlgorithmToken(ForwardEmailBook) -> CommonRecommendationType.ForwardAddressbookTweet, + getAlgorithmToken(ForwardPhoneBook) -> CommonRecommendationType.ForwardAddressbookTweet, + getAlgorithmToken(CrowdSearchAccounts) -> CommonRecommendationType.CrowdSearchTweet + ) + + def getAlgorithmToken(algorithm: Algorithm): Int = { + AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap(algorithm) + } + + def getCRTForAlgoToken(algorithmToken: Int): Option[CommonRecommendationType] = { + crtsByAlgoToken.get(algorithmToken) + } +} + +case class FRSTweetCandidateAdaptor( + crMixerTweetStore: CrMixerTweetStore, + tweetyPieStore: ReadableStore[Long, TweetyPieResult], + tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], + userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], + uttEntityHydrationStore: UttEntityHydrationStore, + topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], + globalStats: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + + private val stats = globalStats.scope(this.getClass.getSimpleName) + private val crtStats = stats.scope("CandidateDistribution") + private val totalRequests = stats.counter("total_requests") + + // Candidate Distribution stats + private val reverseAddressbookCounter = crtStats.counter("reverse_addressbook") + private val forwardAddressbookCounter = crtStats.counter("forward_addressbook") + private val frsTweetCounter = crtStats.counter("frs_tweet") + private val nonReplyTweetsCounter = stats.counter("non_reply_tweets") + private val crtToCounterMapping: Map[CommonRecommendationType, Counter] = Map( + CommonRecommendationType.ReverseAddressbookTweet -> reverseAddressbookCounter, + CommonRecommendationType.ForwardAddressbookTweet -> forwardAddressbookCounter, + CommonRecommendationType.FrsTweet -> frsTweetCounter + ) + + private val emptyTweetyPieResult = stats.stat("empty_tweetypie_result") + + private[this] val numberReturnedCandidates = stats.stat("returned_candidates_from_earlybird") + private[this] val numberCandidateWithTopic: Counter = stats.counter("num_can_with_topic") + private[this] val numberCandidateWithoutTopic: Counter = stats.counter("num_can_without_topic") + + private val userTweetTweetyPieStoreCounter = stats.counter("user_tweet_tweetypie_store") + + override val name: String = this.getClass.getSimpleName + + private def filterInvalidTweets( + tweetIds: Seq[Long], + target: Target + ): Future[Map[Long, TweetyPieResult]] = { + val resMap = { + if (target.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors)) { + userTweetTweetyPieStoreCounter.incr() + val keys = tweetIds.map { tweetId => + UserTweet(tweetId, Some(target.targetId)) + } + userTweetTweetyPieStore + .multiGet(keys.toSet).map { + case (userTweet, resultFut) => + userTweet.tweetId -> resultFut + }.toMap + } else { + (if (target.params(PushFeatureSwitchParams.EnableVFInTweetypie)) { + tweetyPieStore + } else { + tweetyPieStoreNoVF + }).multiGet(tweetIds.toSet) + } + } + + Future.collect(resMap).map { tweetyPieResultMap => + // Filter out replies and generate earlybird candidates only for non-empty tweetypie result + val cands = filterOutReplyTweet(tweetyPieResultMap, nonReplyTweetsCounter).collect { + case (id: Long, Some(result)) => + id -> result + } + + emptyTweetyPieResult.add(tweetyPieResultMap.size - cands.size) + cands + } + } + + private def buildRawCandidates( + target: Target, + ebCandidates: Seq[FRSTweetCandidate] + ): Future[Option[Seq[RawCandidate with TweetCandidate]]] = { + + val enableTopic = target.params(PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicAnnotation) + val topicScoreThre = + target.params(PushFeatureSwitchParams.FrsTweetCandidatesTopicScoreThreshold) + + val ebTweets = ebCandidates.map { ebCandidate => + ebCandidate.tweetId -> ebCandidate.tweetyPieResult + }.toMap + + val tweetIdLocalizedEntityMapFut = TopicsUtil.getTweetIdLocalizedEntityMap( + target, + ebTweets, + uttEntityHydrationStore, + topicSocialProofServiceStore, + enableTopic, + topicScoreThre + ) + + Future.join(target.deviceInfo, tweetIdLocalizedEntityMapFut).map { + case (Some(deviceInfo), tweetIdLocalizedEntityMap) => + val candidates = ebCandidates + .map { ebCandidate => + val crt = ebCandidate.commonRecType + crtToCounterMapping.get(crt).foreach(_.incr()) + + val tweetId = ebCandidate.tweetId + val localizedEntityOpt = { + if (tweetIdLocalizedEntityMap + .contains(tweetId) && tweetIdLocalizedEntityMap.contains( + tweetId) && deviceInfo.isTopicsEligible) { + tweetIdLocalizedEntityMap(tweetId) + } else { + None + } + } + + PushAdaptorUtil.generateOutOfNetworkTweetCandidates( + inputTarget = target, + id = ebCandidate.tweetId, + mediaCRT = MediaCRT( + crt, + crt, + crt + ), + result = ebCandidate.tweetyPieResult, + localizedEntity = localizedEntityOpt) + }.filter { candidate => + // If user only has the topic setting enabled, filter out all non-topic cands + deviceInfo.isRecommendationsEligible || (deviceInfo.isTopicsEligible && candidate.semanticCoreEntityId.nonEmpty) + } + + candidates.map { candidate => + if (candidate.semanticCoreEntityId.nonEmpty) { + numberCandidateWithTopic.incr() + } else { + numberCandidateWithoutTopic.incr() + } + } + + numberReturnedCandidates.add(candidates.length) + Some(candidates) + case _ => Some(Seq.empty) + } + } + + def getTweetCandidatesFromCrMixer( + inputTarget: Target, + showAllResultsFromFrs: Boolean, + ): Future[Option[Seq[RawCandidate with TweetCandidate]]] = { + Future + .join( + inputTarget.seenTweetIds, + inputTarget.pushRecItems, + inputTarget.countryCode, + inputTarget.targetLanguage).flatMap { + case (seenTweetIds, pastRecItems, countryCode, language) => + val pastUserRecs = pastRecItems.userIds.toSeq + val request = FrsTweetRequest( + clientContext = ClientContext( + userId = Some(inputTarget.targetId), + countryCode = countryCode, + languageCode = language + ), + product = Product.Notifications, + productContext = Some(ProductContext.NotificationsContext(NotificationsContext())), + excludedUserIds = Some(pastUserRecs), + excludedTweetIds = Some(seenTweetIds) + ) + crMixerTweetStore.getFRSTweetCandidates(request).flatMap { + case Some(response) => + val tweetIds = response.tweets.map(_.tweetId) + val validTweets = filterInvalidTweets(tweetIds, inputTarget) + validTweets.flatMap { tweetypieMap => + val ebCandidates = response.tweets + .map { frsTweet => + val candidateTweetId = frsTweet.tweetId + val resultFromTweetyPie = tweetypieMap.get(candidateTweetId) + new FRSTweetCandidate { + override val tweetId = candidateTweetId + override val features = None + override val tweetyPieResult = resultFromTweetyPie + override val feedbackToken = frsTweet.frsPrimarySource + override val commonRecType: CommonRecommendationType = feedbackToken + .flatMap(token => + FRSAlgorithmFeedbackTokenUtil.getCRTForAlgoToken(token)).getOrElse( + CommonRecommendationType.FrsTweet) + } + }.filter { ebCandidate => + showAllResultsFromFrs || ebCandidate.commonRecType == CommonRecommendationType.ReverseAddressbookTweet + } + + numberReturnedCandidates.add(ebCandidates.length) + buildRawCandidates( + inputTarget, + ebCandidates + ) + } + case _ => Future.None + } + } + } + + override def get(inputTarget: Target): Future[Option[Seq[RawCandidate with TweetCandidate]]] = { + totalRequests.incr() + val enableResultsFromFrs = + inputTarget.params(PushFeatureSwitchParams.EnableResultFromFrsCandidates) + getTweetCandidatesFromCrMixer(inputTarget, enableResultsFromFrs) + } + + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + lazy val enableFrsCandidates = target.params(PushFeatureSwitchParams.EnableFrsCandidates) + PushDeviceUtil.isRecommendationsEligible(target).flatMap { isEnabledForRecosSetting => + PushDeviceUtil.isTopicsEligible(target).map { topicSettingEnabled => + val isEnabledForTopics = + topicSettingEnabled && target.params( + PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicSetting) + (isEnabledForRecosSetting || isEnabledForTopics) && enableFrsCandidates + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.scala new file mode 100644 index 0000000000..24d0cb64a7 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/GenericCandidateAdaptor.scala @@ -0,0 +1,107 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.candidate._ +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +object GenericCandidates { + type Target = + TargetUser + with UserDetails + with TargetDecider + with TargetABDecider + with TweetImpressionHistory + with HTLVisitHistory + with MaxTweetAge + with NewUserDetails + with FrigateHistory + with TargetWithSeedUsers +} + +case class GenericCandidateAdaptor( + genericCandidates: CandidateSource[GenericCandidates.Target, Candidate], + tweetyPieStore: ReadableStore[Long, TweetyPieResult], + tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], + stats: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + + override val name: String = genericCandidates.name + + private def generateTweetFavCandidate( + _target: Target, + _tweetId: Long, + _socialContextActions: Seq[SocialContextAction], + socialContextActionsAllTypes: Seq[SocialContextAction], + _tweetyPieResult: Option[TweetyPieResult] + ): RawCandidate = { + new RawCandidate with TweetFavoriteCandidate { + override val socialContextActions = _socialContextActions + override val socialContextAllTypeActions = + socialContextActionsAllTypes + val tweetId = _tweetId + val target = _target + val tweetyPieResult = _tweetyPieResult + } + } + + private def generateTweetRetweetCandidate( + _target: Target, + _tweetId: Long, + _socialContextActions: Seq[SocialContextAction], + socialContextActionsAllTypes: Seq[SocialContextAction], + _tweetyPieResult: Option[TweetyPieResult] + ): RawCandidate = { + new RawCandidate with TweetRetweetCandidate { + override val socialContextActions = _socialContextActions + override val socialContextAllTypeActions = socialContextActionsAllTypes + val tweetId = _tweetId + val target = _target + val tweetyPieResult = _tweetyPieResult + } + } + + override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { + genericCandidates.get(inputTarget).map { candidatesOpt => + candidatesOpt + .map { candidates => + val candidatesSeq = + candidates.collect { + case tweetRetweet: TweetRetweetCandidate + if inputTarget.params(PushParams.MRTweetRetweetRecsParam) => + generateTweetRetweetCandidate( + inputTarget, + tweetRetweet.tweetId, + tweetRetweet.socialContextActions, + tweetRetweet.socialContextAllTypeActions, + tweetRetweet.tweetyPieResult) + case tweetFavorite: TweetFavoriteCandidate + if inputTarget.params(PushParams.MRTweetFavRecsParam) => + generateTweetFavCandidate( + inputTarget, + tweetFavorite.tweetId, + tweetFavorite.socialContextActions, + tweetFavorite.socialContextAllTypeActions, + tweetFavorite.tweetyPieResult) + } + candidatesSeq.foreach { candidate => + stats.counter(s"${candidate.commonRecType}_count").incr() + } + candidatesSeq + } + } + } + + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + PushDeviceUtil.isRecommendationsEligible(target).map { isAvailable => + isAvailable && target.params(PushParams.GenericCandidateAdaptorDecider) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.scala new file mode 100644 index 0000000000..37d11535fb --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/HighQualityTweetsAdaptor.scala @@ -0,0 +1,280 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.CandidateSourceEligible +import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.HighQualityCandidateGroupEnum +import com.twitter.frigate.pushservice.params.HighQualityCandidateGroupEnum._ +import com.twitter.frigate.pushservice.params.PushConstants.targetUserAgeFeatureName +import com.twitter.frigate.pushservice.params.PushConstants.targetUserPreferredLanguage +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} +import com.twitter.frigate.pushservice.predicate.TargetPredicates +import com.twitter.frigate.pushservice.util.MediaCRT +import com.twitter.frigate.pushservice.util.PushAdaptorUtil +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.pushservice.util.TopicsUtil +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.interests.thriftscala.InterestId.SemanticCore +import com.twitter.interests.thriftscala.UserInterests +import com.twitter.language.normalization.UserDisplayLanguage +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets +import com.twitter.util.Future + +object HighQualityTweetsHelper { + def getFollowedTopics( + target: Target, + interestsWithLookupContextStore: ReadableStore[ + InterestsLookupRequestWithContext, + UserInterests + ], + followedTopicsStats: Stat + ): Future[Seq[Long]] = { + TopicsUtil + .getTopicsFollowedByUser(target, interestsWithLookupContextStore, followedTopicsStats).map { + userInterestsOpt => + val userInterests = userInterestsOpt.getOrElse(Seq.empty) + val extractedTopicIds = userInterests.flatMap { + _.interestId match { + case SemanticCore(semanticCore) => Some(semanticCore.id) + case _ => None + } + } + extractedTopicIds + } + } + + def getTripQueries( + target: Target, + enabledGroups: Set[HighQualityCandidateGroupEnum.Value], + interestsWithLookupContextStore: ReadableStore[ + InterestsLookupRequestWithContext, + UserInterests + ], + sourceIds: Seq[String], + stat: Stat + ): Future[Set[TripDomain]] = { + + val followedTopicIdsSetFut: Future[Set[Long]] = if (enabledGroups.contains(Topic)) { + getFollowedTopics(target, interestsWithLookupContextStore, stat).map(topicIds => + topicIds.toSet) + } else { + Future.value(Set.empty) + } + + Future + .join(target.featureMap, target.inferredUserDeviceLanguage, followedTopicIdsSetFut).map { + case ( + featureMap, + deviceLanguageOpt, + followedTopicIds + ) => + val ageBucketOpt = if (enabledGroups.contains(AgeBucket)) { + featureMap.categoricalFeatures.get(targetUserAgeFeatureName) + } else { + None + } + + val languageOptions: Set[Option[String]] = if (enabledGroups.contains(Language)) { + val userPreferredLanguages = featureMap.sparseBinaryFeatures + .getOrElse(targetUserPreferredLanguage, Set.empty[String]) + if (userPreferredLanguages.nonEmpty) { + userPreferredLanguages.map(lang => Some(UserDisplayLanguage.toTweetLanguage(lang))) + } else { + Set(deviceLanguageOpt.map(UserDisplayLanguage.toTweetLanguage)) + } + } else Set(None) + + val followedTopicOptions: Set[Option[Long]] = if (followedTopicIds.nonEmpty) { + followedTopicIds.map(topic => Some(topic)) + } else Set(None) + + val tripQueries = followedTopicOptions.flatMap { topicOption => + languageOptions.flatMap { languageOption => + sourceIds.map { sourceId => + TripDomain( + sourceId = sourceId, + language = languageOption, + placeId = None, + topicId = topicOption, + gender = None, + ageBucket = ageBucketOpt + ) + } + } + } + + tripQueries + } + } +} + +case class HighQualityTweetsAdaptor( + tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], + interestsWithLookupContextStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], + tweetyPieStore: ReadableStore[Long, TweetyPieResult], + tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], + globalStats: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + + override def name: String = this.getClass.getSimpleName + + private val stats = globalStats.scope("HighQualityCandidateAdaptor") + private val followedTopicsStats = stats.stat("followed_topics") + private val missingResponseCounter = stats.counter("missing_respond_counter") + private val crtFatigueCounter = stats.counter("fatigue_by_crt") + private val fallbackRequestsCounter = stats.counter("fallback_requests") + + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + PushDeviceUtil.isRecommendationsEligible(target).map { + _ && target.params(FS.HighQualityCandidatesEnableCandidateSource) + } + } + + private val highQualityCandidateFrequencyPredicate = { + TargetPredicates + .pushRecTypeFatiguePredicate( + CommonRecommendationType.TripHqTweet, + FS.HighQualityTweetsPushInterval, + FS.MaxHighQualityTweetsPushGivenInterval, + stats + ) + } + + private def getTripCandidatesStrato( + target: Target + ): Future[Map[Long, Set[TripDomain]]] = { + val tripQueriesF: Future[Set[TripDomain]] = HighQualityTweetsHelper.getTripQueries( + target = target, + enabledGroups = target.params(FS.HighQualityCandidatesEnableGroups).toSet, + interestsWithLookupContextStore = interestsWithLookupContextStore, + sourceIds = target.params(FS.TripTweetCandidateSourceIds), + stat = followedTopicsStats + ) + + lazy val fallbackTripQueriesFut: Future[Set[TripDomain]] = + if (target.params(FS.HighQualityCandidatesEnableFallback)) + HighQualityTweetsHelper.getTripQueries( + target = target, + enabledGroups = target.params(FS.HighQualityCandidatesFallbackEnabledGroups).toSet, + interestsWithLookupContextStore = interestsWithLookupContextStore, + sourceIds = target.params(FS.HighQualityCandidatesFallbackSourceIds), + stat = followedTopicsStats + ) + else Future.value(Set.empty) + + val initialTweetsFut: Future[Map[TripDomain, Seq[TripTweet]]] = tripQueriesF.flatMap { + tripQueries => getTripTweetsByDomains(tripQueries) + } + + val tweetsByDomainFut: Future[Map[TripDomain, Seq[TripTweet]]] = + if (target.params(FS.HighQualityCandidatesEnableFallback)) { + initialTweetsFut.flatMap { candidates => + val minCandidatesForFallback: Int = + target.params(FS.HighQualityCandidatesMinNumOfCandidatesToFallback) + val validCandidates = candidates.filter(_._2.size >= minCandidatesForFallback) + + if (validCandidates.nonEmpty) { + Future.value(validCandidates) + } else { + fallbackTripQueriesFut.flatMap { fallbackTripDomains => + fallbackRequestsCounter.incr(fallbackTripDomains.size) + getTripTweetsByDomains(fallbackTripDomains) + } + } + } + } else { + initialTweetsFut + } + + val numOfCandidates: Int = target.params(FS.HighQualityCandidatesNumberOfCandidates) + tweetsByDomainFut.map(tweetsByDomain => reformatDomainTweetMap(tweetsByDomain, numOfCandidates)) + } + + private def getTripTweetsByDomains( + tripQueries: Set[TripDomain] + ): Future[Map[TripDomain, Seq[TripTweet]]] = { + Future.collect(tripTweetCandidateStore.multiGet(tripQueries)).map { response => + response + .filter(p => p._2.exists(_.tweets.nonEmpty)) + .mapValues(_.map(_.tweets).getOrElse(Seq.empty)) + } + } + + private def reformatDomainTweetMap( + tweetsByDomain: Map[TripDomain, Seq[TripTweet]], + numOfCandidates: Int + ): Map[Long, Set[TripDomain]] = tweetsByDomain + .flatMap { + case (tripDomain, tripTweets) => + tripTweets + .sortBy(_.score)(Ordering[Double].reverse) + .take(numOfCandidates) + .map { tweet => (tweet.tweetId, tripDomain) } + }.groupBy(_._1).mapValues(_.map(_._2).toSet) + + private def buildRawCandidate( + target: Target, + tweetyPieResult: TweetyPieResult, + tripDomain: Option[scala.collection.Set[TripDomain]] + ): RawCandidate = { + PushAdaptorUtil.generateOutOfNetworkTweetCandidates( + inputTarget = target, + id = tweetyPieResult.tweet.id, + mediaCRT = MediaCRT( + CommonRecommendationType.TripHqTweet, + CommonRecommendationType.TripHqTweet, + CommonRecommendationType.TripHqTweet + ), + result = Some(tweetyPieResult), + tripTweetDomain = tripDomain + ) + } + + private def getTweetyPieResults( + target: Target, + tweetToTripDomain: Map[Long, Set[TripDomain]] + ): Future[Map[Long, Option[TweetyPieResult]]] = { + Future.collect((if (target.params(FS.EnableVFInTweetypie)) { + tweetyPieStore + } else { + tweetyPieStoreNoVF + }).multiGet(tweetToTripDomain.keySet)) + } + + override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { + for { + tweetsToTripDomainMap <- getTripCandidatesStrato(target) + tweetyPieResults <- getTweetyPieResults(target, tweetsToTripDomainMap) + } yield { + val candidates = tweetyPieResults.flatMap { + case (tweetId, tweetyPieResultOpt) => + tweetyPieResultOpt.map(buildRawCandidate(target, _, tweetsToTripDomainMap.get(tweetId))) + } + if (candidates.nonEmpty) { + highQualityCandidateFrequencyPredicate(Seq(target)) + .map(_.head) + .map { isTargetFatigueEligible => + if (isTargetFatigueEligible) Some(candidates) + else { + crtFatigueCounter.incr() + None + } + } + + Some(candidates.toSeq) + } else { + missingResponseCounter.incr() + None + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.scala new file mode 100644 index 0000000000..59744b375b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/ListsToRecommendCandidateAdaptor.scala @@ -0,0 +1,152 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.CandidateSourceEligible +import com.twitter.frigate.common.base.ListPushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.predicate.TargetPredicates +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.geoduck.service.thriftscala.LocationResponse +import com.twitter.interests_discovery.thriftscala.DisplayLocation +import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists +import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest +import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +case class ListsToRecommendCandidateAdaptor( + listRecommendationsStore: ReadableStore[String, NonPersonalizedRecommendedLists], + geoDuckV2Store: ReadableStore[Long, LocationResponse], + idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse], + globalStats: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + + override val name: String = this.getClass.getSimpleName + + private[this] val stats = globalStats.scope(name) + private[this] val noLocationCodeCounter = stats.counter("no_location_code") + private[this] val noCandidatesCounter = stats.counter("no_candidates_for_geo") + private[this] val disablePopGeoListsCounter = stats.counter("disable_pop_geo_lists") + private[this] val disableIDSListsCounter = stats.counter("disable_ids_lists") + + private def getListCandidate( + targetUser: Target, + _listId: Long + ): RawCandidate with ListPushCandidate = { + new RawCandidate with ListPushCandidate { + override val listId: Long = _listId + + override val commonRecType: CommonRecommendationType = CommonRecommendationType.List + + override val target: Target = targetUser + } + } + + private def getListsRecommendedFromHistory( + target: Target + ): Future[Seq[Long]] = { + target.history.map { history => + history.sortedHistory.flatMap { + case (_, notif) if notif.commonRecommendationType == List => + notif.listNotification.map(_.listId) + case _ => None + } + } + } + + private def getIDSListRecs( + target: Target, + historicalListIds: Seq[Long] + ): Future[Seq[Long]] = { + val request = RecommendedListsRequest( + target.targetId, + DisplayLocation.ListDiscoveryPage, + Some(historicalListIds) + ) + if (target.params(PushFeatureSwitchParams.EnableIDSListRecommendations)) { + idsStore.get(request).map { + case Some(response) => + response.channels.map(_.id) + case _ => Nil + } + } else { + disableIDSListsCounter.incr() + Future.Nil + } + } + + private def getPopGeoLists( + target: Target, + historicalListIds: Seq[Long] + ): Future[Seq[Long]] = { + if (target.params(PushFeatureSwitchParams.EnablePopGeoListRecommendations)) { + geoDuckV2Store.get(target.targetId).flatMap { + case Some(locationResponse) if locationResponse.geohash.isDefined => + val geoHashLength = + target.params(PushFeatureSwitchParams.ListRecommendationsGeoHashLength) + val geoHash = locationResponse.geohash.get.take(geoHashLength) + listRecommendationsStore + .get(s"geohash_$geoHash") + .map { + case Some(recommendedLists) => + recommendedLists.recommendedListsByAlgo.flatMap { topLists => + topLists.lists.collect { + case list if !historicalListIds.contains(list.listId) => list.listId + } + } + case _ => Nil + } + case _ => + noLocationCodeCounter.incr() + Future.Nil + } + } else { + disablePopGeoListsCounter.incr() + Future.Nil + } + } + + override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { + getListsRecommendedFromHistory(target).flatMap { historicalListIds => + Future + .join( + getPopGeoLists(target, historicalListIds), + getIDSListRecs(target, historicalListIds) + ) + .map { + case (popGeoListsIds, idsListIds) => + val candidates = (idsListIds ++ popGeoListsIds).map(getListCandidate(target, _)) + Some(candidates) + case _ => + noCandidatesCounter.incr() + None + } + } + } + + private val pushCapFatiguePredicate = TargetPredicates.pushRecTypeFatiguePredicate( + CommonRecommendationType.List, + PushFeatureSwitchParams.ListRecommendationsPushInterval, + PushFeatureSwitchParams.MaxListRecommendationsPushGivenInterval, + stats, + ) + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + + val isNotFatigued = pushCapFatiguePredicate.apply(Seq(target)).map(_.head) + + Future + .join( + PushDeviceUtil.isRecommendationsEligible(target), + isNotFatigued + ).map { + case (userRecommendationsEligible, isUnderCAP) => + userRecommendationsEligible && isUnderCAP && target.params( + PushFeatureSwitchParams.EnableListRecommendations) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.scala new file mode 100644 index 0000000000..e5ac0b5164 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/LoggedOutPushCandidateSourceGenerator.scala @@ -0,0 +1,54 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.CandidateSourceEligible +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.geoduck.service.thriftscala.LocationResponse +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets +import com.twitter.content_mixer.thriftscala.ContentMixerRequest +import com.twitter.content_mixer.thriftscala.ContentMixerResponse +import com.twitter.geoduck.common.thriftscala.Location +import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace +import com.twitter.recommendation.interests.discovery.core.model.InterestDomain + +class LoggedOutPushCandidateSourceGenerator( + tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], + geoDuckV2Store: ReadableStore[Long, LocationResponse], + safeCachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], + cachedTweetyPieStoreV2NoVF: ReadableStore[Long, TweetyPieResult], + cachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], + contentMixerStore: ReadableStore[ContentMixerRequest, ContentMixerResponse], + softUserLocationStore: ReadableStore[Long, Location], + topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[(Long, Double)]]], + topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace], +)( + implicit val globalStats: StatsReceiver) { + val sources: Seq[CandidateSource[Target, RawCandidate] with CandidateSourceEligible[ + Target, + RawCandidate + ]] = { + Seq( + TripGeoCandidatesAdaptor( + tripTweetCandidateStore, + contentMixerStore, + safeCachedTweetyPieStoreV2, + cachedTweetyPieStoreV2NoVF, + globalStats + ), + TopTweetsByGeoAdaptor( + geoDuckV2Store, + softUserLocationStore, + topTweetsByGeoStore, + topTweetsByGeoV2VersionedStore, + cachedTweetyPieStoreV2, + cachedTweetyPieStoreV2NoVF, + globalStats + ) + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.scala new file mode 100644 index 0000000000..98568e9dc4 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/OnboardingPushCandidateAdaptor.scala @@ -0,0 +1,101 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.CandidateSourceEligible +import com.twitter.frigate.common.base.DiscoverTwitterCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} +import com.twitter.frigate.pushservice.predicate.DiscoverTwitterPredicate +import com.twitter.frigate.pushservice.predicate.TargetPredicates +import com.twitter.frigate.pushservice.util.PushAppPermissionUtil +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} +import com.twitter.util.Future + +class OnboardingPushCandidateAdaptor( + globalStats: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + + override val name: String = this.getClass.getSimpleName + + private[this] val stats = globalStats.scope(name) + private[this] val requestNum = stats.counter("request_num") + private[this] val addressBookCandNum = stats.counter("address_book_cand_num") + private[this] val completeOnboardingCandNum = stats.counter("complete_onboarding_cand_num") + + private def generateOnboardingPushRawCandidate( + _target: Target, + _commonRecType: CRT + ): RawCandidate = { + new RawCandidate with DiscoverTwitterCandidate { + override val target = _target + override val commonRecType = _commonRecType + } + } + + private def getEligibleCandsForTarget( + target: Target + ): Future[Option[Seq[RawCandidate]]] = { + val addressBookFatigue = + TargetPredicates + .pushRecTypeFatiguePredicate( + CRT.AddressBookUploadPush, + FS.FatigueForOnboardingPushes, + FS.MaxOnboardingPushInInterval, + stats)(Seq(target)).map(_.head) + val completeOnboardingFatigue = + TargetPredicates + .pushRecTypeFatiguePredicate( + CRT.CompleteOnboardingPush, + FS.FatigueForOnboardingPushes, + FS.MaxOnboardingPushInInterval, + stats)(Seq(target)).map(_.head) + + Future + .join( + target.appPermissions, + addressBookFatigue, + completeOnboardingFatigue + ).map { + case (appPermissionOpt, addressBookPredicate, completeOnboardingPredicate) => + val addressBookUploaded = + PushAppPermissionUtil.hasTargetUploadedAddressBook(appPermissionOpt) + val abUploadCandidate = + if (!addressBookUploaded && addressBookPredicate && target.params( + FS.EnableAddressBookPush)) { + addressBookCandNum.incr() + Some(generateOnboardingPushRawCandidate(target, CRT.AddressBookUploadPush)) + } else if (!addressBookUploaded && (completeOnboardingPredicate || + target.params(FS.DisableOnboardingPushFatigue)) && target.params( + FS.EnableCompleteOnboardingPush)) { + completeOnboardingCandNum.incr() + Some(generateOnboardingPushRawCandidate(target, CRT.CompleteOnboardingPush)) + } else None + + val allCandidates = + Seq(abUploadCandidate).filter(_.isDefined).flatten + if (allCandidates.nonEmpty) Some(allCandidates) else None + } + } + + override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { + requestNum.incr() + val minDurationForMRElapsed = + DiscoverTwitterPredicate + .minDurationElapsedSinceLastMrPushPredicate( + name, + FS.MrMinDurationSincePushForOnboardingPushes, + stats)(Seq(inputTarget)).map(_.head) + minDurationForMRElapsed.flatMap { minDurationElapsed => + if (minDurationElapsed) getEligibleCandsForTarget(inputTarget) else Future.None + } + } + + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + PushDeviceUtil + .isRecommendationsEligible(target).map(_ && target.params(FS.EnableOnboardingPushes)) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.scala new file mode 100644 index 0000000000..ea2dcd008c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/PushCandidateSourceGenerator.scala @@ -0,0 +1,162 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.content_mixer.thriftscala.ContentMixerRequest +import com.twitter.content_mixer.thriftscala.ContentMixerResponse +import com.twitter.explore_ranker.thriftscala.ExploreRankerRequest +import com.twitter.explore_ranker.thriftscala.ExploreRankerResponse +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.candidate._ +import com.twitter.frigate.common.store.RecentTweetsQuery +import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.store._ +import com.twitter.geoduck.common.thriftscala.Location +import com.twitter.geoduck.service.thriftscala.LocationResponse +import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.hermit.store.tweetypie.UserTweet +import com.twitter.interests.thriftscala.UserInterests +import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists +import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest +import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse +import com.twitter.recommendation.interests.discovery.core.model.InterestDomain +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets +import com.twitter.tsp.thriftscala.TopicSocialProofRequest +import com.twitter.tsp.thriftscala.TopicSocialProofResponse + +/** + * PushCandidateSourceGenerator generates candidate source list for a given Target user + */ +class PushCandidateSourceGenerator( + earlybirdCandidates: CandidateSource[EarlybirdCandidateSource.Query, EarlybirdCandidate], + userTweetEntityGraphCandidates: CandidateSource[UserTweetEntityGraphCandidates.Target, Candidate], + cachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], + safeCachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult], + userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], + safeUserTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], + cachedTweetyPieStoreV2NoVF: ReadableStore[Long, TweetyPieResult], + edgeStore: ReadableStore[RelationEdge, Boolean], + interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], + uttEntityHydrationStore: UttEntityHydrationStore, + geoDuckV2Store: ReadableStore[Long, LocationResponse], + topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[(Long, Double)]]], + topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace], + tweetImpressionsStore: TweetImpressionsStore, + recommendedTrendsCandidateSource: RecommendedTrendsCandidateSource, + recentTweetsByAuthorStore: ReadableStore[RecentTweetsQuery, Seq[Seq[Long]]], + topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], + crMixerStore: CrMixerTweetStore, + contentMixerStore: ReadableStore[ContentMixerRequest, ContentMixerResponse], + exploreRankerStore: ReadableStore[ExploreRankerRequest, ExploreRankerResponse], + softUserLocationStore: ReadableStore[Long, Location], + tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], + listRecsStore: ReadableStore[String, NonPersonalizedRecommendedLists], + idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse] +)( + implicit val globalStats: StatsReceiver) { + + private val earlyBirdFirstDegreeCandidateAdaptor = EarlyBirdFirstDegreeCandidateAdaptor( + earlybirdCandidates, + cachedTweetyPieStoreV2, + cachedTweetyPieStoreV2NoVF, + userTweetTweetyPieStore, + PushFeatureSwitchParams.NumberOfMaxEarlybirdInNetworkCandidatesParam, + globalStats + ) + + private val frsTweetCandidateAdaptor = FRSTweetCandidateAdaptor( + crMixerStore, + cachedTweetyPieStoreV2, + cachedTweetyPieStoreV2NoVF, + userTweetTweetyPieStore, + uttEntityHydrationStore, + topicSocialProofServiceStore, + globalStats + ) + + private val contentRecommenderMixerAdaptor = ContentRecommenderMixerAdaptor( + crMixerStore, + safeCachedTweetyPieStoreV2, + edgeStore, + topicSocialProofServiceStore, + uttEntityHydrationStore, + globalStats + ) + + private val tripGeoCandidatesAdaptor = TripGeoCandidatesAdaptor( + tripTweetCandidateStore, + contentMixerStore, + safeCachedTweetyPieStoreV2, + cachedTweetyPieStoreV2NoVF, + globalStats + ) + + val sources: Seq[ + CandidateSource[Target, RawCandidate] with CandidateSourceEligible[ + Target, + RawCandidate + ] + ] = { + Seq( + earlyBirdFirstDegreeCandidateAdaptor, + GenericCandidateAdaptor( + userTweetEntityGraphCandidates, + cachedTweetyPieStoreV2, + cachedTweetyPieStoreV2NoVF, + globalStats.scope("UserTweetEntityGraphCandidates") + ), + new OnboardingPushCandidateAdaptor(globalStats), + TopTweetsByGeoAdaptor( + geoDuckV2Store, + softUserLocationStore, + topTweetsByGeoStore, + topTweetsByGeoV2VersionedStore, + cachedTweetyPieStoreV2, + cachedTweetyPieStoreV2NoVF, + globalStats + ), + frsTweetCandidateAdaptor, + TopTweetImpressionsCandidateAdaptor( + recentTweetsByAuthorStore, + cachedTweetyPieStoreV2, + cachedTweetyPieStoreV2NoVF, + tweetImpressionsStore, + globalStats + ), + TrendsCandidatesAdaptor( + softUserLocationStore, + recommendedTrendsCandidateSource, + safeCachedTweetyPieStoreV2, + cachedTweetyPieStoreV2NoVF, + safeUserTweetTweetyPieStore, + globalStats + ), + contentRecommenderMixerAdaptor, + tripGeoCandidatesAdaptor, + HighQualityTweetsAdaptor( + tripTweetCandidateStore, + interestsLookupStore, + cachedTweetyPieStoreV2, + cachedTweetyPieStoreV2NoVF, + globalStats + ), + ExploreVideoTweetCandidateAdaptor( + exploreRankerStore, + cachedTweetyPieStoreV2, + globalStats + ), + ListsToRecommendCandidateAdaptor( + listRecsStore, + geoDuckV2Store, + idsStore, + globalStats + ) + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.scala new file mode 100644 index 0000000000..25ab31e85f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetImpressionsCandidateAdaptor.scala @@ -0,0 +1,326 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.CandidateSourceEligible +import com.twitter.frigate.common.base.TopTweetImpressionsCandidate +import com.twitter.frigate.common.store.RecentTweetsQuery +import com.twitter.frigate.common.util.SnowflakeUtils +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} +import com.twitter.frigate.pushservice.store.TweetImpressionsStore +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.FutureOps +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +case class TweetImpressionsCandidate( + tweetId: Long, + tweetyPieResultOpt: Option[TweetyPieResult], + impressionsCountOpt: Option[Long]) + +case class TopTweetImpressionsCandidateAdaptor( + recentTweetsFromTflockStore: ReadableStore[RecentTweetsQuery, Seq[Seq[Long]]], + tweetyPieStore: ReadableStore[Long, TweetyPieResult], + tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], + tweetImpressionsStore: TweetImpressionsStore, + globalStats: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + + private val stats = globalStats.scope("TopTweetImpressionsAdaptor") + private val tweetImpressionsCandsStat = stats.stat("top_tweet_impressions_cands_dist") + + private val eligibleUsersCounter = stats.counter("eligible_users") + private val noneligibleUsersCounter = stats.counter("noneligible_users") + private val meetsMinTweetsRequiredCounter = stats.counter("meets_min_tweets_required") + private val belowMinTweetsRequiredCounter = stats.counter("below_min_tweets_required") + private val aboveMaxInboundFavoritesCounter = stats.counter("above_max_inbound_favorites") + private val meetsImpressionsRequiredCounter = stats.counter("meets_impressions_required") + private val belowImpressionsRequiredCounter = stats.counter("below_impressions_required") + private val meetsFavoritesThresholdCounter = stats.counter("meets_favorites_threshold") + private val aboveFavoritesThresholdCounter = stats.counter("above_favorites_threshold") + private val emptyImpressionsMapCounter = stats.counter("empty_impressions_map") + + private val tflockResultsStat = stats.stat("tflock", "results") + private val emptyTflockResult = stats.counter("tflock", "empty_result") + private val nonEmptyTflockResult = stats.counter("tflock", "non_empty_result") + + private val originalTweetsStat = stats.stat("tweets", "original_tweets") + private val retweetsStat = stats.stat("tweets", "retweets") + private val allRetweetsOnlyCounter = stats.counter("tweets", "all_retweets_only") + private val allOriginalTweetsOnlyCounter = stats.counter("tweets", "all_original_tweets_only") + + private val emptyTweetypieMap = stats.counter("", "empty_tweetypie_map") + private val emptyTweetyPieResult = stats.stat("", "empty_tweetypie_result") + private val allEmptyTweetypieResults = stats.counter("", "all_empty_tweetypie_results") + + private val eligibleUsersAfterImpressionsFilter = + stats.counter("eligible_users_after_impressions_filter") + private val eligibleUsersAfterFavoritesFilter = + stats.counter("eligible_users_after_favorites_filter") + private val eligibleUsersWithEligibleTweets = + stats.counter("eligible_users_with_eligible_tweets") + + private val eligibleTweetCands = stats.stat("eligible_tweet_cands") + private val getCandsRequestCounter = + stats.counter("top_tweet_impressions_get_request") + + override val name: String = this.getClass.getSimpleName + + override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { + getCandsRequestCounter.incr() + val eligibleCandidatesFut = getTweetImpressionsCandidates(inputTarget) + eligibleCandidatesFut.map { eligibleCandidates => + if (eligibleCandidates.nonEmpty) { + eligibleUsersWithEligibleTweets.incr() + eligibleTweetCands.add(eligibleCandidates.size) + val candidate = getMostImpressionsTweet(eligibleCandidates) + Some( + Seq( + generateTopTweetImpressionsCandidate( + inputTarget, + candidate.tweetId, + candidate.tweetyPieResultOpt, + candidate.impressionsCountOpt.getOrElse(0L)))) + } else None + } + } + + private def getTweetImpressionsCandidates( + inputTarget: Target + ): Future[Seq[TweetImpressionsCandidate]] = { + val originalTweets = getRecentOriginalTweetsForUser(inputTarget) + originalTweets.flatMap { tweetyPieResultsMap => + val numDaysSearchForOriginalTweets = + inputTarget.params(FS.TopTweetImpressionsOriginalTweetsNumDaysSearch) + val moreRecentTweetIds = + getMoreRecentTweetIds(tweetyPieResultsMap.keySet.toSeq, numDaysSearchForOriginalTweets) + val isEligible = isEligibleUser(inputTarget, tweetyPieResultsMap, moreRecentTweetIds) + if (isEligible) filterByEligibility(inputTarget, tweetyPieResultsMap, moreRecentTweetIds) + else Future.Nil + } + } + + private def getRecentOriginalTweetsForUser( + targetUser: Target + ): Future[Map[Long, TweetyPieResult]] = { + val tweetyPieResultsMapFut = getTflockStoreResults(targetUser).flatMap { recentTweetIds => + FutureOps.mapCollect((targetUser.params(FS.EnableVFInTweetypie) match { + case true => tweetyPieStore + case false => tweetyPieStoreNoVF + }).multiGet(recentTweetIds.toSet)) + } + tweetyPieResultsMapFut.map { tweetyPieResultsMap => + if (tweetyPieResultsMap.isEmpty) { + emptyTweetypieMap.incr() + Map.empty + } else removeRetweets(tweetyPieResultsMap) + } + } + + private def getTflockStoreResults(targetUser: Target): Future[Seq[Long]] = { + val maxResults = targetUser.params(FS.TopTweetImpressionsRecentTweetsByAuthorStoreMaxResults) + val maxAge = targetUser.params(FS.TopTweetImpressionsTotalFavoritesLimitNumDaysSearch) + val recentTweetsQuery = + RecentTweetsQuery( + userIds = Seq(targetUser.targetId), + maxResults = maxResults, + maxAge = maxAge.days + ) + recentTweetsFromTflockStore + .get(recentTweetsQuery).map { + case Some(tweetIdsAll) => + val tweetIds = tweetIdsAll.headOption.getOrElse(Seq.empty) + val numTweets = tweetIds.size + if (numTweets > 0) { + tflockResultsStat.add(numTweets) + nonEmptyTflockResult.incr() + } else emptyTflockResult.incr() + tweetIds + case _ => Nil + } + } + + private def removeRetweets( + tweetyPieResultsMap: Map[Long, Option[TweetyPieResult]] + ): Map[Long, TweetyPieResult] = { + val nonEmptyTweetyPieResults: Map[Long, TweetyPieResult] = tweetyPieResultsMap.collect { + case (key, Some(value)) => (key, value) + } + emptyTweetyPieResult.add(tweetyPieResultsMap.size - nonEmptyTweetyPieResults.size) + + if (nonEmptyTweetyPieResults.nonEmpty) { + val originalTweets = nonEmptyTweetyPieResults.filter { + case (_, tweetyPieResult) => + tweetyPieResult.sourceTweet.isEmpty + } + val numOriginalTweets = originalTweets.size + val numRetweets = nonEmptyTweetyPieResults.size - originalTweets.size + originalTweetsStat.add(numOriginalTweets) + retweetsStat.add(numRetweets) + if (numRetweets == 0) allOriginalTweetsOnlyCounter.incr() + if (numOriginalTweets == 0) allRetweetsOnlyCounter.incr() + originalTweets + } else { + allEmptyTweetypieResults.incr() + Map.empty + } + } + + private def getMoreRecentTweetIds( + tweetIds: Seq[Long], + numDays: Int + ): Seq[Long] = { + tweetIds.filter { tweetId => + SnowflakeUtils.isRecent(tweetId, numDays.days) + } + } + + private def isEligibleUser( + inputTarget: Target, + tweetyPieResults: Map[Long, TweetyPieResult], + recentTweetIds: Seq[Long] + ): Boolean = { + val minNumTweets = inputTarget.params(FS.TopTweetImpressionsMinNumOriginalTweets) + lazy val totalFavoritesLimit = + inputTarget.params(FS.TopTweetImpressionsTotalInboundFavoritesLimit) + if (recentTweetIds.size >= minNumTweets) { + meetsMinTweetsRequiredCounter.incr() + val isUnderLimit = isUnderTotalInboundFavoritesLimit(tweetyPieResults, totalFavoritesLimit) + if (isUnderLimit) eligibleUsersCounter.incr() + else { + aboveMaxInboundFavoritesCounter.incr() + noneligibleUsersCounter.incr() + } + isUnderLimit + } else { + belowMinTweetsRequiredCounter.incr() + noneligibleUsersCounter.incr() + false + } + } + + private def getFavoriteCounts( + tweetyPieResult: TweetyPieResult + ): Long = tweetyPieResult.tweet.counts.flatMap(_.favoriteCount).getOrElse(0L) + + private def isUnderTotalInboundFavoritesLimit( + tweetyPieResults: Map[Long, TweetyPieResult], + totalFavoritesLimit: Long + ): Boolean = { + val favoritesIterator = tweetyPieResults.valuesIterator.map(getFavoriteCounts) + val totalInboundFavorites = favoritesIterator.sum + totalInboundFavorites <= totalFavoritesLimit + } + + def filterByEligibility( + inputTarget: Target, + tweetyPieResults: Map[Long, TweetyPieResult], + tweetIds: Seq[Long] + ): Future[Seq[TweetImpressionsCandidate]] = { + lazy val minNumImpressions: Long = inputTarget.params(FS.TopTweetImpressionsMinRequired) + lazy val maxNumLikes: Long = inputTarget.params(FS.TopTweetImpressionsMaxFavoritesPerTweet) + for { + filteredImpressionsMap <- getFilteredImpressionsMap(tweetIds, minNumImpressions) + tweetIdsFilteredByFavorites <- + getTweetIdsFilteredByFavorites(filteredImpressionsMap.keySet, tweetyPieResults, maxNumLikes) + } yield { + if (filteredImpressionsMap.nonEmpty) eligibleUsersAfterImpressionsFilter.incr() + if (tweetIdsFilteredByFavorites.nonEmpty) eligibleUsersAfterFavoritesFilter.incr() + + val candidates = tweetIdsFilteredByFavorites.map { tweetId => + TweetImpressionsCandidate( + tweetId, + tweetyPieResults.get(tweetId), + filteredImpressionsMap.get(tweetId)) + } + tweetImpressionsCandsStat.add(candidates.length) + candidates + } + } + + private def getFilteredImpressionsMap( + tweetIds: Seq[Long], + minNumImpressions: Long + ): Future[Map[Long, Long]] = { + getImpressionsCounts(tweetIds).map { impressionsMap => + if (impressionsMap.isEmpty) emptyImpressionsMapCounter.incr() + impressionsMap.filter { + case (_, numImpressions) => + val isValid = numImpressions >= minNumImpressions + if (isValid) { + meetsImpressionsRequiredCounter.incr() + } else { + belowImpressionsRequiredCounter.incr() + } + isValid + } + } + } + + private def getTweetIdsFilteredByFavorites( + filteredTweetIds: Set[Long], + tweetyPieResults: Map[Long, TweetyPieResult], + maxNumLikes: Long + ): Future[Seq[Long]] = { + val filteredByFavoritesTweetIds = filteredTweetIds.filter { tweetId => + val tweetyPieResultOpt = tweetyPieResults.get(tweetId) + val isValid = tweetyPieResultOpt.exists { tweetyPieResult => + getFavoriteCounts(tweetyPieResult) <= maxNumLikes + } + if (isValid) meetsFavoritesThresholdCounter.incr() + else aboveFavoritesThresholdCounter.incr() + isValid + } + Future(filteredByFavoritesTweetIds.toSeq) + } + + private def getMostImpressionsTweet( + filteredResults: Seq[TweetImpressionsCandidate] + ): TweetImpressionsCandidate = { + val maxImpressions: Long = filteredResults.map { + _.impressionsCountOpt.getOrElse(0L) + }.max + + val mostImpressionsCandidates: Seq[TweetImpressionsCandidate] = + filteredResults.filter(_.impressionsCountOpt.getOrElse(0L) == maxImpressions) + + mostImpressionsCandidates.maxBy(_.tweetId) + } + + private def getImpressionsCounts( + tweetIds: Seq[Long] + ): Future[Map[Long, Long]] = { + val impressionCountMap = tweetIds.map { tweetId => + tweetId -> tweetImpressionsStore + .getCounts(tweetId).map(_.getOrElse(0L)) + }.toMap + Future.collect(impressionCountMap) + } + + private def generateTopTweetImpressionsCandidate( + inputTarget: Target, + _tweetId: Long, + result: Option[TweetyPieResult], + _impressionsCount: Long + ): RawCandidate = { + new RawCandidate with TopTweetImpressionsCandidate { + override val target: Target = inputTarget + override val tweetId: Long = _tweetId + override val tweetyPieResult: Option[TweetyPieResult] = result + override val impressionsCount: Long = _impressionsCount + } + } + + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + val enabledTopTweetImpressionsNotification = + target.params(FS.EnableTopTweetImpressionsNotification) + + PushDeviceUtil + .isRecommendationsEligible(target).map(_ && enabledTopTweetImpressionsNotification) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.scala new file mode 100644 index 0000000000..3228760fdc --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TopTweetsByGeoAdaptor.scala @@ -0,0 +1,413 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.CandidateSourceEligible +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.model.PushTypes +import com.twitter.frigate.pushservice.params.PopGeoTweetVersion +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.params.TopTweetsForGeoCombination +import com.twitter.frigate.pushservice.params.TopTweetsForGeoRankingFunction +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} +import com.twitter.frigate.pushservice.predicate.DiscoverTwitterPredicate +import com.twitter.frigate.pushservice.predicate.TargetPredicates +import com.twitter.frigate.pushservice.util.MediaCRT +import com.twitter.frigate.pushservice.util.PushAdaptorUtil +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.geoduck.common.thriftscala.{Location => GeoLocation} +import com.twitter.geoduck.service.thriftscala.LocationResponse +import com.twitter.gizmoduck.thriftscala.UserType +import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace +import com.twitter.recommendation.interests.discovery.core.model.InterestDomain +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.FutureOps +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future +import com.twitter.util.Time +import scala.collection.Map + +case class PlaceTweetScore(place: String, tweetId: Long, score: Double) { + def toTweetScore: (Long, Double) = (tweetId, score) +} +case class TopTweetsByGeoAdaptor( + geoduckStoreV2: ReadableStore[Long, LocationResponse], + softUserGeoLocationStore: ReadableStore[Long, GeoLocation], + topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[(Long, Double)]]], + topTweetsByGeoStoreV2: ReadableStore[String, PopTweetsInPlace], + tweetyPieStore: ReadableStore[Long, TweetyPieResult], + tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], + globalStats: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + + override def name: String = this.getClass.getSimpleName + + private[this] val stats = globalStats.scope("TopTweetsByGeoAdaptor") + private[this] val noGeohashUserCounter: Counter = stats.counter("users_with_no_geohash_counter") + private[this] val incomingRequestCounter: Counter = stats.counter("incoming_request_counter") + private[this] val incomingLoggedOutRequestCounter: Counter = + stats.counter("incoming_logged_out_request_counter") + private[this] val loggedOutRawCandidatesCounter = + stats.counter("logged_out_raw_candidates_counter") + private[this] val emptyLoggedOutRawCandidatesCounter = + stats.counter("logged_out_empty_raw_candidates") + private[this] val outputTopTweetsByGeoCounter: Stat = + stats.stat("output_top_tweets_by_geo_counter") + private[this] val loggedOutPopByGeoV2CandidatesCounter: Counter = + stats.counter("logged_out_pop_by_geo_candidates") + private[this] val dormantUsersSince14DaysCounter: Counter = + stats.counter("dormant_user_since_14_days_counter") + private[this] val dormantUsersSince30DaysCounter: Counter = + stats.counter("dormant_user_since_30_days_counter") + private[this] val nonDormantUsersSince14DaysCounter: Counter = + stats.counter("non_dormant_user_since_14_days_counter") + private[this] val topTweetsByGeoTake100Counter: Counter = + stats.counter("top_tweets_by_geo_take_100_counter") + private[this] val combinationRequestsCounter = + stats.scope("combination_method_request_counter") + private[this] val popGeoTweetVersionCounter = + stats.scope("popgeo_tweet_version_counter") + private[this] val nonReplyTweetsCounter = stats.counter("non_reply_tweets") + + val MaxGeoHashSize = 4 + + private def constructKeys( + geohash: Option[String], + accountCountryCode: Option[String], + keyLengths: Seq[Int], + version: PopGeoTweetVersion.Value + ): Set[String] = { + val geohashKeys = geohash match { + case Some(hash) => keyLengths.map { version + "_geohash_" + hash.take(_) } + case _ => Seq.empty + } + + val accountCountryCodeKeys = + accountCountryCode.toSeq.map(version + "_country_" + _.toUpperCase) + (geohashKeys ++ accountCountryCodeKeys).toSet + } + + def convertToPlaceTweetScore( + popTweetsInPlace: Seq[PopTweetsInPlace] + ): Seq[PlaceTweetScore] = { + popTweetsInPlace.flatMap { + case p => + p.popTweets.map { + case popTweet => PlaceTweetScore(p.place, popTweet.tweetId, popTweet.score) + } + } + } + + def sortGeoHashTweets( + placeTweetScores: Seq[PlaceTweetScore], + rankingFunction: TopTweetsForGeoRankingFunction.Value + ): Seq[PlaceTweetScore] = { + rankingFunction match { + case TopTweetsForGeoRankingFunction.Score => + placeTweetScores.sortBy(_.score)(Ordering[Double].reverse) + case TopTweetsForGeoRankingFunction.GeohashLengthAndThenScore => + placeTweetScores + .sortBy(row => (row.place.length, row.score))(Ordering[(Int, Double)].reverse) + } + } + + def getResultsForLambdaStore( + inputTarget: Target, + geohash: Option[String], + store: ReadableStore[String, PopTweetsInPlace], + topk: Int, + version: PopGeoTweetVersion.Value + ): Future[Seq[(Long, Double)]] = { + inputTarget.accountCountryCode.flatMap { countryCode => + val keys = { + if (inputTarget.params(FS.EnableCountryCodeBackoffTopTweetsByGeo)) + constructKeys(geohash, countryCode, inputTarget.params(FS.GeoHashLengthList), version) + else + constructKeys(geohash, None, inputTarget.params(FS.GeoHashLengthList), version) + } + FutureOps + .mapCollect(store.multiGet(keys)).map { + case geohashTweetMap => + val popTweets = + geohashTweetMap.values.flatten.toSeq + val results = sortGeoHashTweets( + convertToPlaceTweetScore(popTweets), + inputTarget.params(FS.RankingFunctionForTopTweetsByGeo)) + .map(_.toTweetScore).take(topk) + results + } + } + } + + def getPopGeoTweetsForLoggedOutUsers( + inputTarget: Target, + store: ReadableStore[String, PopTweetsInPlace] + ): Future[Seq[(Long, Double)]] = { + inputTarget.countryCode.flatMap { countryCode => + val keys = constructKeys(None, countryCode, Seq(4), PopGeoTweetVersion.Prod) + FutureOps.mapCollect(store.multiGet(keys)).map { + case tweetMap => + val tweets = tweetMap.values.flatten.toSeq + loggedOutPopByGeoV2CandidatesCounter.incr(tweets.size) + val popTweets = sortGeoHashTweets( + convertToPlaceTweetScore(tweets), + TopTweetsForGeoRankingFunction.Score).map(_.toTweetScore) + popTweets + } + } + } + + def getRankedTweets( + inputTarget: Target, + geohash: Option[String] + ): Future[Seq[(Long, Double)]] = { + val MaxTopTweetsByGeoCandidatesToTake = + inputTarget.params(FS.MaxTopTweetsByGeoCandidatesToTake) + val scoringFn: String = inputTarget.params(FS.ScoringFuncForTopTweetsByGeo) + val combinationMethod = inputTarget.params(FS.TopTweetsByGeoCombinationParam) + val popGeoTweetVersion = inputTarget.params(FS.PopGeoTweetVersionParam) + + inputTarget.isHeavyUserState.map { isHeavyUser => + stats + .scope(combinationMethod.toString).scope(popGeoTweetVersion.toString).scope( + "IsHeavyUser_" + isHeavyUser.toString).counter().incr() + } + combinationRequestsCounter.scope(combinationMethod.toString).counter().incr() + popGeoTweetVersionCounter.scope(popGeoTweetVersion.toString).counter().incr() + lazy val geoStoreResults = if (geohash.isDefined) { + val hash = geohash.get.take(MaxGeoHashSize) + topTweetsByGeoStore + .get( + InterestDomain[String](hash) + ) + .map { + case Some(scoringFnToTweetsMapOpt) => + val tweetsWithScore = scoringFnToTweetsMapOpt + .getOrElse(scoringFn, List.empty) + val sortedResults = sortGeoHashTweets( + tweetsWithScore.map { + case (tweetId, score) => PlaceTweetScore(hash, tweetId, score) + }, + TopTweetsForGeoRankingFunction.Score + ).map(_.toTweetScore).take( + MaxTopTweetsByGeoCandidatesToTake + ) + sortedResults + case _ => Seq.empty + } + } else Future.value(Seq.empty) + lazy val versionPopGeoTweetResults = + getResultsForLambdaStore( + inputTarget, + geohash, + topTweetsByGeoStoreV2, + MaxTopTweetsByGeoCandidatesToTake, + popGeoTweetVersion + ) + combinationMethod match { + case TopTweetsForGeoCombination.Default => geoStoreResults + case TopTweetsForGeoCombination.AccountsTweetFavAsBackfill => + Future.join(geoStoreResults, versionPopGeoTweetResults).map { + case (geoStoreTweets, versionPopGeoTweets) => + (geoStoreTweets ++ versionPopGeoTweets).take(MaxTopTweetsByGeoCandidatesToTake) + } + case TopTweetsForGeoCombination.AccountsTweetFavIntermixed => + Future.join(geoStoreResults, versionPopGeoTweetResults).map { + case (geoStoreTweets, versionPopGeoTweets) => + CandidateSource.interleaveSeqs(Seq(geoStoreTweets, versionPopGeoTweets)) + } + } + } + + override def get(inputTarget: Target): Future[Option[Seq[RawCandidate]]] = { + if (inputTarget.isLoggedOutUser) { + incomingLoggedOutRequestCounter.incr() + val rankedTweets = getPopGeoTweetsForLoggedOutUsers(inputTarget, topTweetsByGeoStoreV2) + val rawCandidates = { + rankedTweets.map { rt => + FutureOps + .mapCollect( + tweetyPieStore + .multiGet(rt.map { case (tweetId, _) => tweetId }.toSet)) + .map { tweetyPieResultMap => + val results = buildTopTweetsByGeoRawCandidates( + inputTarget, + None, + tweetyPieResultMap + ) + if (results.isEmpty) { + emptyLoggedOutRawCandidatesCounter.incr() + } + loggedOutRawCandidatesCounter.incr(results.size) + Some(results) + } + }.flatten + } + rawCandidates + } else { + incomingRequestCounter.incr() + getGeoHashForUsers(inputTarget).flatMap { geohash => + if (geohash.isEmpty) noGeohashUserCounter.incr() + getRankedTweets(inputTarget, geohash).map { rt => + if (rt.size == 100) { + topTweetsByGeoTake100Counter.incr(1) + } + FutureOps + .mapCollect((inputTarget.params(FS.EnableVFInTweetypie) match { + case true => tweetyPieStore + case false => tweetyPieStoreNoVF + }).multiGet(rt.map { case (tweetId, _) => tweetId }.toSet)) + .map { tweetyPieResultMap => + Some( + buildTopTweetsByGeoRawCandidates( + inputTarget, + None, + filterOutReplyTweet( + tweetyPieResultMap, + nonReplyTweetsCounter + ) + ) + ) + } + }.flatten + } + } + } + + private def getGeoHashForUsers( + inputTarget: Target + ): Future[Option[String]] = { + + inputTarget.targetUser.flatMap { + case Some(user) => + user.userType match { + case UserType.Soft => + softUserGeoLocationStore + .get(inputTarget.targetId) + .map(_.flatMap(_.geohash.flatMap(_.stringGeohash))) + + case _ => + geoduckStoreV2.get(inputTarget.targetId).map(_.flatMap(_.geohash)) + } + + case None => Future.None + } + } + + private def buildTopTweetsByGeoRawCandidates( + target: PushTypes.Target, + locationName: Option[String], + topTweets: Map[Long, Option[TweetyPieResult]] + ): Seq[RawCandidate with TweetCandidate] = { + val candidates = topTweets.map { tweetIdTweetyPieResultMap => + PushAdaptorUtil.generateOutOfNetworkTweetCandidates( + inputTarget = target, + id = tweetIdTweetyPieResultMap._1, + mediaCRT = MediaCRT( + CommonRecommendationType.GeoPopTweet, + CommonRecommendationType.GeoPopTweet, + CommonRecommendationType.GeoPopTweet + ), + result = tweetIdTweetyPieResultMap._2, + localizedEntity = None + ) + }.toSeq + outputTopTweetsByGeoCounter.add(candidates.length) + candidates + } + + private val topTweetsByGeoFrequencyPredicate = { + TargetPredicates + .pushRecTypeFatiguePredicate( + CommonRecommendationType.GeoPopTweet, + FS.TopTweetsByGeoPushInterval, + FS.MaxTopTweetsByGeoPushGivenInterval, + stats + ) + } + + def getAvailabilityForDormantUser(target: Target): Future[Boolean] = { + lazy val isDormantUserNotFatigued = topTweetsByGeoFrequencyPredicate(Seq(target)).map(_.head) + lazy val enableTopTweetsByGeoForDormantUsers = + target.params(FS.EnableTopTweetsByGeoCandidatesForDormantUsers) + + target.lastHTLVisitTimestamp.flatMap { + case Some(lastHTLTimestamp) => + val minTimeSinceLastLogin = + target.params(FS.MinimumTimeSinceLastLoginForGeoPopTweetPush).ago + val timeSinceInactive = target.params(FS.TimeSinceLastLoginForGeoPopTweetPush).ago + val lastActiveTimestamp = Time.fromMilliseconds(lastHTLTimestamp) + if (lastActiveTimestamp > minTimeSinceLastLogin) { + nonDormantUsersSince14DaysCounter.incr() + Future.False + } else { + dormantUsersSince14DaysCounter.incr() + isDormantUserNotFatigued.map { isUserNotFatigued => + lastActiveTimestamp < timeSinceInactive && + enableTopTweetsByGeoForDormantUsers && + isUserNotFatigued + } + } + case _ => + dormantUsersSince30DaysCounter.incr() + isDormantUserNotFatigued.map { isUserNotFatigued => + enableTopTweetsByGeoForDormantUsers && isUserNotFatigued + } + } + } + + def getAvailabilityForPlaybookSetUp(target: Target): Future[Boolean] = { + lazy val enableTopTweetsByGeoForNewUsers = target.params(FS.EnableTopTweetsByGeoCandidates) + val isTargetEligibleForMrFatigueCheck = target.isAccountAtleastNDaysOld( + target.params(FS.MrMinDurationSincePushForTopTweetsByGeoPushes)) + val isMrFatigueCheckEnabled = + target.params(FS.EnableMrMinDurationSinceMrPushFatigue) + val applyPredicateForTopTweetsByGeo = + if (isMrFatigueCheckEnabled) { + if (isTargetEligibleForMrFatigueCheck) { + DiscoverTwitterPredicate + .minDurationElapsedSinceLastMrPushPredicate( + name, + FS.MrMinDurationSincePushForTopTweetsByGeoPushes, + stats + ).andThen( + topTweetsByGeoFrequencyPredicate + )(Seq(target)).map(_.head) + } else { + Future.False + } + } else { + topTweetsByGeoFrequencyPredicate(Seq(target)).map(_.head) + } + applyPredicateForTopTweetsByGeo.map { predicateResult => + predicateResult && enableTopTweetsByGeoForNewUsers + } + } + + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + if (target.isLoggedOutUser) { + Future.True + } else { + PushDeviceUtil + .isRecommendationsEligible(target).map( + _ && target.params(PushParams.PopGeoCandidatesDecider)).flatMap { isAvailable => + if (isAvailable) { + Future + .join(getAvailabilityForDormantUser(target), getAvailabilityForPlaybookSetUp(target)) + .map { + case (isAvailableForDormantUser, isAvailableForPlaybook) => + isAvailableForDormantUser || isAvailableForPlaybook + case _ => false + } + } else Future.False + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.scala new file mode 100644 index 0000000000..4e7ec33148 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TrendsCandidatesAdaptor.scala @@ -0,0 +1,215 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.events.recos.thriftscala.DisplayLocation +import com.twitter.events.recos.thriftscala.TrendsContext +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.CandidateSourceEligible +import com.twitter.frigate.common.base.TrendTweetCandidate +import com.twitter.frigate.common.base.TrendsCandidate +import com.twitter.frigate.common.candidate.RecommendedTrendsCandidateSource +import com.twitter.frigate.common.candidate.RecommendedTrendsCandidateSource.Query +import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.adaptor.TrendsCandidatesAdaptor._ +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.predicate.TargetPredicates +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.geoduck.common.thriftscala.Location +import com.twitter.gizmoduck.thriftscala.UserType +import com.twitter.hermit.store.tweetypie.UserTweet +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future +import scala.collection.Map + +object TrendsCandidatesAdaptor { + type TweetId = Long + type EventId = Long +} + +case class TrendsCandidatesAdaptor( + softUserGeoLocationStore: ReadableStore[Long, Location], + recommendedTrendsCandidateSource: RecommendedTrendsCandidateSource, + tweetyPieStore: ReadableStore[Long, TweetyPieResult], + tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], + safeUserTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult], + statsReceiver: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + override val name = this.getClass.getSimpleName + + private val trendAdaptorStats = statsReceiver.scope("TrendsCandidatesAdaptor") + private val trendTweetCandidateNumber = trendAdaptorStats.counter("trend_tweet_candidate") + private val nonReplyTweetsCounter = trendAdaptorStats.counter("non_reply_tweets") + + private def getQuery(target: Target): Future[Query] = { + def getUserCountryCode(target: Target): Future[Option[String]] = { + target.targetUser.flatMap { + case Some(user) if user.userType == UserType.Soft => + softUserGeoLocationStore + .get(user.id) + .map(_.flatMap(_.simpleRgcResult.flatMap(_.countryCodeAlpha2))) + + case _ => target.accountCountryCode + } + } + + for { + countryCode <- getUserCountryCode(target) + inferredLanguage <- target.inferredUserDeviceLanguage + } yield { + Query( + userId = target.targetId, + displayLocation = DisplayLocation.MagicRecs, + languageCode = inferredLanguage, + countryCode = countryCode, + maxResults = target.params(PushFeatureSwitchParams.MaxRecommendedTrendsToQuery) + ) + } + } + + /** + * Query candidates only if sent at most [[PushFeatureSwitchParams.MaxTrendTweetNotificationsInDuration]] + * trend tweet notifications in [[PushFeatureSwitchParams.TrendTweetNotificationsFatigueDuration]] + */ + val trendTweetFatiguePredicate = TargetPredicates.pushRecTypeFatiguePredicate( + CommonRecommendationType.TrendTweet, + PushFeatureSwitchParams.TrendTweetNotificationsFatigueDuration, + PushFeatureSwitchParams.MaxTrendTweetNotificationsInDuration, + trendAdaptorStats + ) + + private val recommendedTrendsWithTweetsCandidateSource: CandidateSource[ + Target, + RawCandidate with TrendsCandidate + ] = recommendedTrendsCandidateSource + .convert[Target, TrendsCandidate]( + getQuery, + recommendedTrendsCandidateSource.identityCandidateMapper + ) + .batchMapValues[Target, RawCandidate with TrendsCandidate]( + trendsCandidatesToTweetCandidates(_, _, getTweetyPieResults)) + + private def getTweetyPieResults( + tweetIds: Seq[TweetId], + target: Target + ): Future[Map[TweetId, TweetyPieResult]] = { + if (target.params(PushFeatureSwitchParams.EnableSafeUserTweetTweetypieStore)) { + Future + .collect( + safeUserTweetTweetyPieStore.multiGet( + tweetIds.toSet.map(UserTweet(_, Some(target.targetId))))).map { + _.collect { + case (userTweet, Some(tweetyPieResult)) => userTweet.tweetId -> tweetyPieResult + } + } + } else { + Future + .collect((target.params(PushFeatureSwitchParams.EnableVFInTweetypie) match { + case true => tweetyPieStore + case false => tweetyPieStoreNoVF + }).multiGet(tweetIds.toSet)).map { tweetyPieResultMap => + filterOutReplyTweet(tweetyPieResultMap, nonReplyTweetsCounter).collect { + case (tweetId, Some(tweetyPieResult)) => tweetId -> tweetyPieResult + } + } + } + } + + /** + * + * @param _target: [[Target]] object representing notificaion recipient user + * @param trendsCandidates: Sequence of [[TrendsCandidate]] returned from ERS + * @return: Seq of trends candidates expanded to associated tweets. + */ + private def trendsCandidatesToTweetCandidates( + _target: Target, + trendsCandidates: Seq[TrendsCandidate], + getTweetyPieResults: (Seq[TweetId], Target) => Future[Map[TweetId, TweetyPieResult]] + ): Future[Seq[RawCandidate with TrendsCandidate]] = { + + def generateTrendTweetCandidates( + trendCandidate: TrendsCandidate, + tweetyPieResults: Map[TweetId, TweetyPieResult] + ) = { + val tweetIds = trendCandidate.context.curatedRepresentativeTweets.getOrElse(Seq.empty) ++ + trendCandidate.context.algoRepresentativeTweets.getOrElse(Seq.empty) + + tweetIds.flatMap { tweetId => + tweetyPieResults.get(tweetId).map { _tweetyPieResult => + new RawCandidate with TrendTweetCandidate { + override val trendId: String = trendCandidate.trendId + override val trendName: String = trendCandidate.trendName + override val landingUrl: String = trendCandidate.landingUrl + override val timeBoundedLandingUrl: Option[String] = + trendCandidate.timeBoundedLandingUrl + override val context: TrendsContext = trendCandidate.context + override val tweetyPieResult: Option[TweetyPieResult] = Some(_tweetyPieResult) + override val tweetId: TweetId = _tweetyPieResult.tweet.id + override val target: Target = _target + } + } + } + } + + // collect all tweet ids associated with all trends + val allTweetIds = trendsCandidates.flatMap { trendsCandidate => + val context = trendsCandidate.context + context.curatedRepresentativeTweets.getOrElse(Seq.empty) ++ + context.algoRepresentativeTweets.getOrElse(Seq.empty) + } + + getTweetyPieResults(allTweetIds, _target) + .map { tweetIdToTweetyPieResult => + val trendTweetCandidates = trendsCandidates.flatMap { trendCandidate => + val allTrendTweetCandidates = generateTrendTweetCandidates( + trendCandidate, + tweetIdToTweetyPieResult + ) + + val (tweetCandidatesFromCuratedTrends, tweetCandidatesFromNonCuratedTrends) = + allTrendTweetCandidates.partition(_.isCuratedTrend) + + tweetCandidatesFromCuratedTrends.filter( + _.target.params(PushFeatureSwitchParams.EnableCuratedTrendTweets)) ++ + tweetCandidatesFromNonCuratedTrends.filter( + _.target.params(PushFeatureSwitchParams.EnableNonCuratedTrendTweets)) + } + + trendTweetCandidateNumber.incr(trendTweetCandidates.size) + trendTweetCandidates + } + } + + /** + * + * @param target: [[Target]] user + * @return: true if customer is eligible to receive trend tweet notifications + * + */ + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + PushDeviceUtil + .isRecommendationsEligible(target) + .map(target.params(PushParams.TrendsCandidateDecider) && _) + } + + override def get(target: Target): Future[Option[Seq[RawCandidate with TrendsCandidate]]] = { + recommendedTrendsWithTweetsCandidateSource + .get(target) + .flatMap { + case Some(candidates) if candidates.nonEmpty => + trendTweetFatiguePredicate(Seq(target)) + .map(_.head) + .map { isTargetFatigueEligible => + if (isTargetFatigueEligible) Some(candidates) + else None + } + + case _ => Future.None + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.scala new file mode 100644 index 0000000000..2bdef162c8 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/adaptor/TripGeoCandidatesAdaptor.scala @@ -0,0 +1,188 @@ +package com.twitter.frigate.pushservice.adaptor + +import com.twitter.content_mixer.thriftscala.ContentMixerProductResponse +import com.twitter.content_mixer.thriftscala.ContentMixerRequest +import com.twitter.content_mixer.thriftscala.ContentMixerResponse +import com.twitter.content_mixer.thriftscala.NotificationsTripTweetsProductContext +import com.twitter.content_mixer.thriftscala.Product +import com.twitter.content_mixer.thriftscala.ProductContext +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.CandidateSourceEligible +import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.filterOutReplyTweet +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.util.MediaCRT +import com.twitter.frigate.pushservice.util.PushAdaptorUtil +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.geoduck.util.country.CountryInfo +import com.twitter.product_mixer.core.thriftscala.ClientContext +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets +import com.twitter.util.Future + +case class TripGeoCandidatesAdaptor( + tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets], + contentMixerStore: ReadableStore[ContentMixerRequest, ContentMixerResponse], + tweetyPieStore: ReadableStore[Long, TweetyPieResult], + tweetyPieStoreNoVF: ReadableStore[Long, TweetyPieResult], + statsReceiver: StatsReceiver) + extends CandidateSource[Target, RawCandidate] + with CandidateSourceEligible[Target, RawCandidate] { + + override def name: String = this.getClass.getSimpleName + + private val stats = statsReceiver.scope(name.stripSuffix("$")) + + private val contentMixerRequests = stats.counter("getTripCandidatesContentMixerRequests") + private val loggedOutTripTweetIds = stats.counter("logged_out_trip_tweet_ids_count") + private val loggedOutRawCandidates = stats.counter("logged_out_raw_candidates_count") + private val rawCandidates = stats.counter("raw_candidates_count") + private val loggedOutEmptyplaceId = stats.counter("logged_out_empty_place_id_count") + private val loggedOutPlaceId = stats.counter("logged_out_place_id_count") + private val nonReplyTweetsCounter = stats.counter("non_reply_tweets") + + override def isCandidateSourceAvailable(target: Target): Future[Boolean] = { + if (target.isLoggedOutUser) { + Future.True + } else { + for { + isRecommendationsSettingEnabled <- PushDeviceUtil.isRecommendationsEligible(target) + inferredLanguage <- target.inferredUserDeviceLanguage + } yield { + isRecommendationsSettingEnabled && + inferredLanguage.nonEmpty && + target.params(PushParams.TripGeoTweetCandidatesDecider) + } + } + + } + + private def buildRawCandidate(target: Target, tweetyPieResult: TweetyPieResult): RawCandidate = { + PushAdaptorUtil.generateOutOfNetworkTweetCandidates( + inputTarget = target, + id = tweetyPieResult.tweet.id, + mediaCRT = MediaCRT( + CommonRecommendationType.TripGeoTweet, + CommonRecommendationType.TripGeoTweet, + CommonRecommendationType.TripGeoTweet + ), + result = Some(tweetyPieResult), + localizedEntity = None + ) + } + + override def get(target: Target): Future[Option[Seq[RawCandidate]]] = { + if (target.isLoggedOutUser) { + for { + tripTweetIds <- getTripCandidatesForLoggedOutTarget(target) + tweetyPieResults <- Future.collect(tweetyPieStoreNoVF.multiGet(tripTweetIds)) + } yield { + val candidates = tweetyPieResults.values.flatten.map(buildRawCandidate(target, _)) + if (candidates.nonEmpty) { + loggedOutRawCandidates.incr(candidates.size) + Some(candidates.toSeq) + } else None + } + } else { + for { + tripTweetIds <- getTripCandidatesContentMixer(target) + tweetyPieResults <- + Future.collect((target.params(PushFeatureSwitchParams.EnableVFInTweetypie) match { + case true => tweetyPieStore + case false => tweetyPieStoreNoVF + }).multiGet(tripTweetIds)) + } yield { + val nonReplyTweets = filterOutReplyTweet(tweetyPieResults, nonReplyTweetsCounter) + val candidates = nonReplyTweets.values.flatten.map(buildRawCandidate(target, _)) + if (candidates.nonEmpty && target.params( + PushFeatureSwitchParams.TripTweetCandidateReturnEnable)) { + rawCandidates.incr(candidates.size) + Some(candidates.toSeq) + } else None + } + } + } + + private def getTripCandidatesContentMixer( + target: Target + ): Future[Set[Long]] = { + contentMixerRequests.incr() + Future + .join( + target.inferredUserDeviceLanguage, + target.deviceInfo + ) + .flatMap { + case (languageOpt, deviceInfoOpt) => + contentMixerStore + .get( + ContentMixerRequest( + clientContext = ClientContext( + userId = Some(target.targetId), + languageCode = languageOpt, + userAgent = deviceInfoOpt.flatMap(_.guessedPrimaryDeviceUserAgent.map(_.toString)) + ), + product = Product.NotificationsTripTweets, + productContext = Some( + ProductContext.NotificationsTripTweetsProductContext( + NotificationsTripTweetsProductContext() + )), + cursor = None, + maxResults = + Some(target.params(PushFeatureSwitchParams.TripTweetMaxTotalCandidates)) + ) + ).map { + _.map { rawResponse => + val tripResponse = + rawResponse.contentMixerProductResponse + .asInstanceOf[ + ContentMixerProductResponse.NotificationsTripTweetsProductResponse] + .notificationsTripTweetsProductResponse + + tripResponse.results.map(_.tweetResult.tweetId).toSet + }.getOrElse(Set.empty) + } + } + } + + private def getTripCandidatesForLoggedOutTarget( + target: Target + ): Future[Set[Long]] = { + Future.join(target.targetLanguage, target.countryCode).flatMap { + case (Some(lang), Some(country)) => + val placeId = CountryInfo.lookupByCode(country).map(_.placeIdLong) + if (placeId.nonEmpty) { + loggedOutPlaceId.incr() + } else { + loggedOutEmptyplaceId.incr() + } + val tripSource = "TOP_GEO_V3_LR" + val tripQuery = TripDomain( + sourceId = tripSource, + language = Some(lang), + placeId = placeId, + topicId = None + ) + val response = tripTweetCandidateStore.get(tripQuery) + val tripTweetIds = + response.map { res => + if (res.isDefined) { + res.get.tweets + .sortBy(_.score)(Ordering[Double].reverse).map(_.tweetId).toSet + } else { + Set.empty[Long] + } + } + tripTweetIds.map { ids => loggedOutTripTweetIds.incr(ids.size) } + tripTweetIds + + case (_, _) => Future.value(Set.empty) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.scala new file mode 100644 index 0000000000..3a0e1dc70f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/Config.scala @@ -0,0 +1,461 @@ +package com.twitter.frigate.pushservice.config + +import com.twitter.abdecider.LoggingABDecider +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse +import com.twitter.audience_rewards.thriftscala.HasSuperFollowingRelationshipRequest +import com.twitter.channels.common.thriftscala.ApiList +import com.twitter.datatools.entityservice.entities.sports.thriftscala._ +import com.twitter.decider.Decider +import com.twitter.discovery.common.configapi.ConfigParamsBuilder +import com.twitter.escherbird.common.thriftscala.QualifiedId +import com.twitter.escherbird.metadata.thriftscala.EntityMegadata +import com.twitter.eventbus.client.EventBusPublisher +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.candidate._ +import com.twitter.frigate.common.history._ +import com.twitter.frigate.common.ml.base._ +import com.twitter.frigate.common.ml.feature._ +import com.twitter.frigate.common.store._ +import com.twitter.frigate.common.store.deviceinfo.DeviceInfo +import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext +import com.twitter.frigate.common.store.interests.UserId +import com.twitter.frigate.common.util._ +import com.twitter.frigate.data_pipeline.features_common._ +import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryKey +import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue +import com.twitter.frigate.dau_model.thriftscala.DauProbability +import com.twitter.frigate.magic_events.thriftscala.FanoutEvent +import com.twitter.frigate.pushcap.thriftscala.PushcapUserHistory +import com.twitter.frigate.pushservice.ml._ +import com.twitter.frigate.pushservice.params.DeciderKey +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushFeatureSwitches +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.send_handler.SendHandlerPushCandidateHydrator +import com.twitter.frigate.pushservice.refresh_handler.PushCandidateHydrator +import com.twitter.frigate.pushservice.store._ +import com.twitter.frigate.pushservice.store.{Ibis2Store => PushIbis2Store} +import com.twitter.frigate.pushservice.take.NotificationServiceRequest +import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe +import com.twitter.frigate.scribe.thriftscala.NotificationScribe +import com.twitter.frigate.thriftscala._ +import com.twitter.frigate.user_states.thriftscala.MRUserHmmState +import com.twitter.geoduck.common.thriftscala.{Location => GeoLocation} +import com.twitter.geoduck.service.thriftscala.LocationResponse +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.hermit.predicate.tweetypie.Perspective +import com.twitter.hermit.predicate.tweetypie.UserTweet +import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery +import com.twitter.hermit.store.tweetypie.{UserTweet => TweetyPieUserTweet} +import com.twitter.hermit.stp.thriftscala.STPResult +import com.twitter.hss.api.thriftscala.UserHealthSignalResponse +import com.twitter.interests.thriftscala.InterestId +import com.twitter.interests.thriftscala.{UserInterests => Interests} +import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists +import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest +import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse +import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} +import com.twitter.ml.api.thriftscala.{DataRecord => ThriftDataRecord} +import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient +import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue +import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStore +import com.twitter.notificationservice.scribe.manhattan.GenericNotificationsFeedbackRequest +import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse +import com.twitter.nrel.heavyranker.CandidateFeatureHydrator +import com.twitter.nrel.heavyranker.{FeatureHydrator => MRFeatureHydrator} +import com.twitter.nrel.heavyranker.{TargetFeatureHydrator => RelevanceTargetFeatureHydrator} +import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment +import com.twitter.permissions_storage.thriftscala.AppPermission +import com.twitter.recommendation.interests.discovery.core.model.InterestDomain +import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendTweetEntityRequest +import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendTweetEntityResponse +import com.twitter.recos.user_user_graph.thriftscala.RecommendUserRequest +import com.twitter.recos.user_user_graph.thriftscala.RecommendUserResponse +import com.twitter.rux.common.strato.thriftscala.UserTargetingProperty +import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWProducer +import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation +import com.twitter.search.common.features.thriftscala.ThriftSearchResultFeatures +import com.twitter.search.earlybird.thriftscala.EarlybirdRequest +import com.twitter.search.earlybird.thriftscala.ThriftSearchResult +import com.twitter.service.gen.scarecrow.thriftscala.Event +import com.twitter.service.gen.scarecrow.thriftscala.TieredActionResult +import com.twitter.service.metastore.gen.thriftscala.Location +import com.twitter.service.metastore.gen.thriftscala.UserLanguages +import com.twitter.servo.decider.DeciderGateBuilder +import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.strato.columns.frigate.logged_out_web_notifications.thriftscala.LOWebNotificationMetadata +import com.twitter.strato.columns.notifications.thriftscala.SourceDestUserRequest +import com.twitter.strato.client.{UserId => StratoUserId} +import com.twitter.timelines.configapi +import com.twitter.timelines.configapi.CompositeConfig +import com.twitter.timelinescorer.thriftscala.v1.ScoredTweet +import com.twitter.topiclisting.TopicListing +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets +import com.twitter.tsp.thriftscala.TopicSocialProofRequest +import com.twitter.tsp.thriftscala.TopicSocialProofResponse +import com.twitter.ubs.thriftscala.SellerTrack +import com.twitter.ubs.thriftscala.AudioSpace +import com.twitter.ubs.thriftscala.Participants +import com.twitter.ubs.thriftscala.SellerApplicationState +import com.twitter.user_session_store.thriftscala.UserSession +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.wtf.scalding.common.thriftscala.UserFeatures + +trait Config { + self => + + def isServiceLocal: Boolean + + def localConfigRepoPath: String + + def inMemCacheOff: Boolean + + def historyStore: PushServiceHistoryStore + + def emailHistoryStore: PushServiceHistoryStore + + def strongTiesStore: ReadableStore[Long, STPResult] + + def safeUserStore: ReadableStore[Long, User] + + def deviceInfoStore: ReadableStore[Long, DeviceInfo] + + def edgeStore: ReadableStore[RelationEdge, Boolean] + + def socialGraphServiceProcessStore: ReadableStore[RelationEdge, Boolean] + + def userUtcOffsetStore: ReadableStore[Long, Duration] + + def cachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult] + + def safeCachedTweetyPieStoreV2: ReadableStore[Long, TweetyPieResult] + + def userTweetTweetyPieStore: ReadableStore[TweetyPieUserTweet, TweetyPieResult] + + def safeUserTweetTweetyPieStore: ReadableStore[TweetyPieUserTweet, TweetyPieResult] + + def cachedTweetyPieStoreV2NoVF: ReadableStore[Long, TweetyPieResult] + + def tweetContentFeatureCacheStore: ReadableStore[Long, ThriftDataRecord] + + def scarecrowCheckEventStore: ReadableStore[Event, TieredActionResult] + + def userTweetPerspectiveStore: ReadableStore[UserTweet, Perspective] + + def userCountryStore: ReadableStore[Long, Location] + + def pushInfoStore: ReadableStore[Long, UserForPushTargeting] + + def loggedOutPushInfoStore: ReadableStore[Long, LOWebNotificationMetadata] + + def tweetImpressionStore: ReadableStore[Long, Seq[Long]] + + def audioSpaceStore: ReadableStore[String, AudioSpace] + + def basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate] + + def baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate] + + def cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate] + + def soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate] + + def nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate] + + def topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse] + + def spaceDeviceFollowStore: ReadableStore[SourceDestUserRequest, Boolean] + + def audioSpaceParticipantsStore: ReadableStore[String, Participants] + + def notificationServiceSender: ReadableStore[ + NotificationServiceRequest, + CreateGenericNotificationResponse + ] + + def ocfFatigueStore: ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] + + def dauProbabilityStore: ReadableStore[Long, DauProbability] + + def hydratedLabeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue] + + def userHTLLastVisitStore: ReadableStore[Long, Seq[Long]] + + def userLanguagesStore: ReadableStore[Long, UserLanguages] + + def topTweetsByGeoStore: ReadableStore[InterestDomain[String], Map[String, List[ + (Long, Double) + ]]] + + def topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace] + + lazy val pushRecItemStore: ReadableStore[PushRecItemsKey, RecItems] = PushRecItemStore( + hydratedLabeledPushRecsStore + ) + + lazy val labeledPushRecsVerifyingStore: ReadableStore[ + LabeledPushRecsVerifyingStoreKey, + LabeledPushRecsVerifyingStoreResponse + ] = + LabeledPushRecsVerifyingStore( + hydratedLabeledPushRecsStore, + historyStore + ) + + lazy val labeledPushRecsDecideredStore: ReadableStore[LabeledPushRecsStoreKey, UserHistoryValue] = + LabeledPushRecsDecideredStore( + labeledPushRecsVerifyingStore, + useHydratedLabeledSendsForFeaturesDeciderKey, + verifyHydratedLabeledSendsForFeaturesDeciderKey + ) + + def onlineUserHistoryStore: ReadableStore[OnlineUserHistoryKey, UserHistoryValue] + + def nsfwConsumerStore: ReadableStore[Long, NSFWUserSegmentation] + + def nsfwProducerStore: ReadableStore[Long, NSFWProducer] + + def popGeoLists: ReadableStore[String, NonPersonalizedRecommendedLists] + + def listAPIStore: ReadableStore[Long, ApiList] + + def openedPushByHourAggregatedStore: ReadableStore[Long, Map[Int, Int]] + + def userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse] + + def reactivatedUserInfoStore: ReadableStore[Long, String] + + def weightedOpenOrNtabClickModelScorer: PushMLModelScorer + + def optoutModelScorer: PushMLModelScorer + + def filteringModelScorer: PushMLModelScorer + + def recentFollowsStore: ReadableStore[Long, Seq[Long]] + + def geoDuckV2Store: ReadableStore[UserId, LocationResponse] + + def realGraphScoresTop500InStore: ReadableStore[Long, Map[Long, Double]] + + def tweetEntityGraphStore: ReadableStore[ + RecommendTweetEntityRequest, + RecommendTweetEntityResponse + ] + + def userUserGraphStore: ReadableStore[RecommendUserRequest, RecommendUserResponse] + + def userFeaturesStore: ReadableStore[Long, UserFeatures] + + def userTargetingPropertyStore: ReadableStore[Long, UserTargetingProperty] + + def timelinesUserSessionStore: ReadableStore[Long, UserSession] + + def optOutUserInterestsStore: ReadableStore[UserId, Seq[InterestId]] + + def ntabCaretFeedbackStore: ReadableStore[GenericNotificationsFeedbackRequest, Seq[ + CaretFeedbackDetails + ]] + + def genericFeedbackStore: ReadableStore[FeedbackRequest, Seq[ + FeedbackPromptValue + ]] + + def genericNotificationFeedbackStore: GenericFeedbackStore + + def semanticCoreMegadataStore: ReadableStore[ + SemanticEntityForQuery, + EntityMegadata + ] + + def tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] + + def earlybirdFeatureStore: ReadableStore[Long, ThriftSearchResultFeatures] + + def earlybirdFeatureBuilder: FeatureBuilder[Long] + + // Feature builders + + def tweetAuthorLocationFeatureBuilder: FeatureBuilder[Location] + + def tweetAuthorLocationFeatureBuilderById: FeatureBuilder[Long] + + def socialContextActionsFeatureBuilder: FeatureBuilder[SocialContextActions] + + def tweetContentFeatureBuilder: FeatureBuilder[Long] + + def tweetAuthorRecentRealGraphFeatureBuilder: FeatureBuilder[RealGraphEdge] + + def socialContextRecentRealGraphFeatureBuilder: FeatureBuilder[Set[RealGraphEdge]] + + def tweetSocialProofFeatureBuilder: FeatureBuilder[TweetSocialProofKey] + + def targetUserFullRealGraphFeatureBuilder: FeatureBuilder[TargetFullRealGraphFeatureKey] + + def postProcessingFeatureBuilder: PostProcessingFeatureBuilder + + def mrOfflineUserCandidateSparseAggregatesFeatureBuilder: FeatureBuilder[ + OfflineSparseAggregateKey + ] + + def mrOfflineUserAggregatesFeatureBuilder: FeatureBuilder[Long] + + def mrOfflineUserCandidateAggregatesFeatureBuilder: FeatureBuilder[OfflineAggregateKey] + + def tweetAnnotationsFeatureBuilder: FeatureBuilder[Long] + + def targetUserMediaRepresentationFeatureBuilder: FeatureBuilder[Long] + + def targetLevelFeatureBuilder: FeatureBuilder[MrRequestContextForFeatureStore] + + def candidateLevelFeatureBuilder: FeatureBuilder[EntityRequestContextForFeatureStore] + + def targetFeatureHydrator: RelevanceTargetFeatureHydrator + + def useHydratedLabeledSendsForFeaturesDeciderKey: String = + DeciderKey.useHydratedLabeledSendsForFeaturesDeciderKey.toString + + def verifyHydratedLabeledSendsForFeaturesDeciderKey: String = + DeciderKey.verifyHydratedLabeledSendsForFeaturesDeciderKey.toString + + def lexServiceStore: ReadableStore[EventRequest, LiveEvent] + + def userMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] + + def producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] + + def mrUserStatePredictionStore: ReadableStore[Long, MRUserHmmState] + + def pushcapDynamicPredictionStore: ReadableStore[Long, PushcapUserHistory] + + def earlybirdCandidateSource: EarlybirdCandidateSource + + def earlybirdSearchStore: ReadableStore[EarlybirdRequest, Seq[ThriftSearchResult]] + + def earlybirdSearchDest: String + + def pushserviceThriftClientId: ClientId + + def simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities] + + def fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent] + + /** + * PostRanking Feature Store Client + */ + def postRankingFeatureStoreClient: DynamicFeatureStoreClient[MrRequestContextForFeatureStore] + + /** + * ReadableStore to fetch [[UserInterests]] from INTS service + */ + def interestsWithLookupContextStore: ReadableStore[InterestsLookupRequestWithContext, Interests] + + /** + * + * @return: [[TopicListing]] object to fetch paused topics and scope from productId + */ + def topicListing: TopicListing + + /** + * + * @return: [[UttEntityHydrationStore]] object + */ + def uttEntityHydrationStore: UttEntityHydrationStore + + def appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission] + + lazy val userTweetEntityGraphCandidates: UserTweetEntityGraphCandidates = + UserTweetEntityGraphCandidates( + cachedTweetyPieStoreV2, + tweetEntityGraphStore, + PushParams.UTEGTweetCandidateSourceParam, + PushFeatureSwitchParams.NumberOfMaxUTEGCandidatesQueriedParam, + PushParams.AllowOneSocialProofForTweetInUTEGParam, + PushParams.OutNetworkTweetsOnlyForUTEGParam, + PushFeatureSwitchParams.MaxTweetAgeParam + )(statsReceiver) + + def pushSendEventBusPublisher: EventBusPublisher[NotificationScribe] + + // miscs. + + def isProd: Boolean + + implicit def statsReceiver: StatsReceiver + + def decider: Decider + + def abDecider: LoggingABDecider + + def casLock: CasLock + + def pushIbisV2Store: PushIbis2Store + + // scribe + def notificationScribe(data: NotificationScribe): Unit + + def requestScribe(data: PushRequestScribe): Unit + + def init(): Future[Unit] = Future.Done + + def configParamsBuilder: ConfigParamsBuilder + + def candidateFeatureHydrator: CandidateFeatureHydrator + + def featureHydrator: MRFeatureHydrator + + def candidateHydrator: PushCandidateHydrator + + def sendHandlerCandidateHydrator: SendHandlerPushCandidateHydrator + + lazy val overridesConfig: configapi.Config = { + val pushFeatureSwitchConfigs: configapi.Config = PushFeatureSwitches( + deciderGateBuilder = new DeciderGateBuilder(decider), + statsReceiver = statsReceiver + ).config + + new CompositeConfig(Seq(pushFeatureSwitchConfigs)) + } + + def realTimeClientEventStore: RealTimeClientEventStore + + def inlineActionHistoryStore: ReadableStore[Long, Seq[(Long, String)]] + + def softUserGeoLocationStore: ReadableStore[Long, GeoLocation] + + def tweetTranslationStore: ReadableStore[TweetTranslationStore.Key, TweetTranslationStore.Value] + + def tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets] + + def softUserFollowingStore: ReadableStore[User, Seq[Long]] + + def superFollowEligibilityUserStore: ReadableStore[Long, Boolean] + + def superFollowCreatorTweetCountStore: ReadableStore[StratoUserId, Int] + + def hasSuperFollowingRelationshipStore: ReadableStore[ + HasSuperFollowingRelationshipRequest, + Boolean + ] + + def superFollowApplicationStatusStore: ReadableStore[(Long, SellerTrack), SellerApplicationState] + + def recentHistoryCacheClient: RecentHistoryCacheClient + + def openAppUserStore: ReadableStore[Long, Boolean] + + def loggedOutHistoryStore: PushServiceHistoryStore + + def idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse] + + def htlScoreStore(userId: Long): ReadableStore[Long, ScoredTweet] +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.scala new file mode 100644 index 0000000000..8d6e95a673 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/DeployConfig.scala @@ -0,0 +1,2150 @@ +package com.twitter.frigate.pushservice.config + +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse +import com.twitter.audience_rewards.thriftscala.HasSuperFollowingRelationshipRequest +import com.twitter.bijection.scrooge.BinaryScalaCodec +import com.twitter.bijection.scrooge.CompactScalaCodec +import com.twitter.channels.common.thriftscala.ApiList +import com.twitter.channels.common.thriftscala.ApiListDisplayLocation +import com.twitter.channels.common.thriftscala.ApiListView +import com.twitter.content_mixer.thriftscala.ContentMixer +import com.twitter.conversions.DurationOps._ +import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService +import com.twitter.cr_mixer.thriftscala.CrMixer +import com.twitter.datatools.entityservice.entities.sports.thriftscala.BaseballGameLiveUpdate +import com.twitter.datatools.entityservice.entities.sports.thriftscala.BasketballGameLiveUpdate +import com.twitter.datatools.entityservice.entities.sports.thriftscala.CricketMatchLiveUpdate +import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate +import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate +import com.twitter.discovery.common.configapi.ConfigParamsBuilder +import com.twitter.discovery.common.configapi.FeatureContextBuilder +import com.twitter.discovery.common.environment.{Environment => NotifEnvironment} +import com.twitter.escherbird.common.thriftscala.Domains +import com.twitter.escherbird.common.thriftscala.QualifiedId +import com.twitter.escherbird.metadata.thriftscala.EntityMegadata +import com.twitter.escherbird.metadata.thriftscala.MetadataService +import com.twitter.escherbird.util.metadatastitch.MetadataStitchClient +import com.twitter.escherbird.util.uttclient +import com.twitter.escherbird.util.uttclient.CacheConfigV2 +import com.twitter.escherbird.util.uttclient.CachedUttClientV2 +import com.twitter.escherbird.utt.strato.thriftscala.Environment +import com.twitter.eventbus.client.EventBusPublisherBuilder +import com.twitter.events.recos.thriftscala.EventsRecosService +import com.twitter.explore_ranker.thriftscala.ExploreRanker +import com.twitter.featureswitches.v2.FeatureSwitches +import com.twitter.finagle.Memcached +import com.twitter.finagle.ThriftMux +import com.twitter.finagle.client.BackupRequestFilter +import com.twitter.finagle.client.ClientRegistry +import com.twitter.finagle.loadbalancer.Balancers +import com.twitter.finagle.memcached.Client +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.mtls.client.MtlsStackClient._ +import com.twitter.finagle.mux.transport.OpportunisticTls +import com.twitter.finagle.service.Retries +import com.twitter.finagle.service.RetryPolicy +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.finagle.thrift.RichClientParam +import com.twitter.finagle.util.DefaultTimer +import com.twitter.flockdb.client._ +import com.twitter.flockdb.client.thriftscala.FlockDB +import com.twitter.frigate.common.base.RandomRanker +import com.twitter.frigate.common.candidate._ +import com.twitter.frigate.common.config.RateLimiterGenerator +import com.twitter.frigate.common.entity_graph_client.RecommendedTweetEntitiesStore +import com.twitter.frigate.common.filter.DynamicRequestMeterFilter +import com.twitter.frigate.common.history._ +import com.twitter.frigate.common.ml.feature._ +import com.twitter.frigate.common.store._ +import com.twitter.frigate.common.store.deviceinfo.DeviceInfoStore +import com.twitter.frigate.common.store.deviceinfo.MobileSdkStore +import com.twitter.frigate.common.store.interests._ +import com.twitter.frigate.common.store.strato.StratoFetchableStore +import com.twitter.frigate.common.store.strato.StratoScannableStore +import com.twitter.frigate.common.util.Finagle.readOnlyThriftService +import com.twitter.frigate.common.util._ +import com.twitter.frigate.data_pipeline.features_common.FeatureStoreUtil +import com.twitter.frigate.data_pipeline.features_common._ +import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryKey +import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue +import com.twitter.frigate.dau_model.thriftscala.DauProbability +import com.twitter.frigate.magic_events.thriftscala.FanoutEvent +import com.twitter.frigate.pushcap.thriftscala.PushcapUserHistory +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.adaptor.LoggedOutPushCandidateSourceGenerator +import com.twitter.frigate.pushservice.adaptor.PushCandidateSourceGenerator +import com.twitter.frigate.pushservice.config.mlconfig.DeepbirdV2ModelConfig +import com.twitter.frigate.pushservice.ml._ +import com.twitter.frigate.pushservice.params._ +import com.twitter.frigate.pushservice.rank.LoggedOutRanker +import com.twitter.frigate.pushservice.rank.RFPHLightRanker +import com.twitter.frigate.pushservice.rank.RFPHRanker +import com.twitter.frigate.pushservice.rank.SubscriptionCreatorRanker +import com.twitter.frigate.pushservice.refresh_handler._ +import com.twitter.frigate.pushservice.refresh_handler.cross.CandidateCopyExpansion +import com.twitter.frigate.pushservice.send_handler.SendHandlerPushCandidateHydrator +import com.twitter.frigate.pushservice.store._ +import com.twitter.frigate.pushservice.take.CandidateNotifier +import com.twitter.frigate.pushservice.take.NotificationSender +import com.twitter.frigate.pushservice.take.NotificationServiceRequest +import com.twitter.frigate.pushservice.take.NotificationServiceSender +import com.twitter.frigate.pushservice.take.NtabOnlyChannelSelector +import com.twitter.frigate.pushservice.take.history.EventBusWriter +import com.twitter.frigate.pushservice.take.history.HistoryWriter +import com.twitter.frigate.pushservice.take.sender.Ibis2Sender +import com.twitter.frigate.pushservice.take.sender.NtabSender +import com.twitter.frigate.pushservice.take.LoggedOutRefreshForPushNotifier +import com.twitter.frigate.pushservice.util.RFPHTakeStepUtil +import com.twitter.frigate.pushservice.util.SendHandlerPredicateUtil +import com.twitter.frigate.scribe.thriftscala.NotificationScribe +import com.twitter.frigate.thriftscala._ +import com.twitter.frigate.user_states.thriftscala.MRUserHmmState +import com.twitter.geoduck.backend.hydration.thriftscala.Hydration +import com.twitter.geoduck.common.thriftscala.PlaceQueryFields +import com.twitter.geoduck.common.thriftscala.PlaceType +import com.twitter.geoduck.common.thriftscala.{Location => GeoLocation} +import com.twitter.geoduck.service.common.clientmodules.GeoduckUserLocate +import com.twitter.geoduck.service.common.clientmodules.GeoduckUserLocateModule +import com.twitter.geoduck.service.thriftscala.LocationResponse +import com.twitter.geoduck.thriftscala.LocationService +import com.twitter.gizmoduck.context.thriftscala.ReadConfig +import com.twitter.gizmoduck.context.thriftscala.TestUserConfig +import com.twitter.gizmoduck.testusers.client.TestUserClientBuilder +import com.twitter.gizmoduck.thriftscala.LookupContext +import com.twitter.gizmoduck.thriftscala.QueryFields +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.gizmoduck.thriftscala.UserService +import com.twitter.hermit.pop_geo.thriftscala.PopTweetsInPlace +import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate +import com.twitter.hermit.predicate.tweetypie.PerspectiveReadableStore +import com.twitter.hermit.store._ +import com.twitter.hermit.store.common._ +import com.twitter.hermit.store.gizmoduck.GizmoduckUserStore +import com.twitter.hermit.store.metastore.UserCountryStore +import com.twitter.hermit.store.metastore.UserLanguagesStore +import com.twitter.hermit.store.scarecrow.ScarecrowCheckEventStore +import com.twitter.hermit.store.semantic_core.MetaDataReadableStore +import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery +import com.twitter.hermit.store.timezone.GizmoduckUserUtcOffsetStore +import com.twitter.hermit.store.timezone.UtcOffsetStore +import com.twitter.hermit.store.tweetypie.TweetyPieStore +import com.twitter.hermit.store.tweetypie.UserTweet +import com.twitter.hermit.store.user_htl_session_store.UserHTLLastVisitReadableStore +import com.twitter.hermit.stp.thriftscala.STPResult +import com.twitter.hss.api.thriftscala.UserHealthSignal +import com.twitter.hss.api.thriftscala.UserHealthSignal._ +import com.twitter.hss.api.thriftscala.UserHealthSignalResponse +import com.twitter.interests.thriftscala.InterestId +import com.twitter.interests.thriftscala.InterestsThriftService +import com.twitter.interests.thriftscala.{UserInterests => Interests} +import com.twitter.interests_discovery.thriftscala.InterestsDiscoveryService +import com.twitter.interests_discovery.thriftscala.NonPersonalizedRecommendedLists +import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest +import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse +import com.twitter.kujaku.domain.thriftscala.MachineTranslationResponse +import com.twitter.livevideo.timeline.client.v2.LiveVideoTimelineClient +import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} +import com.twitter.livevideo.timeline.thrift.thriftscala.TimelineService +import com.twitter.logging.Logger +import com.twitter.ml.api.thriftscala.{DataRecord => ThriftDataRecord} +import com.twitter.ml.featurestore.catalog.entities.core.{Author => TweetAuthorEntity} +import com.twitter.ml.featurestore.catalog.entities.core.{User => TargetUserEntity} +import com.twitter.ml.featurestore.catalog.entities.core.{UserAuthor => UserAuthorEntity} +import com.twitter.ml.featurestore.catalog.entities.magicrecs.{SocialContext => SocialContextEntity} +import com.twitter.ml.featurestore.catalog.entities.magicrecs.{UserSocialContext => TargetUserSocialContextEntity} +import com.twitter.ml.featurestore.timelines.thriftscala.TimelineScorerScoreView +import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest +import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue +import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStore +import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStoreBuilder +import com.twitter.notificationservice.scribe.manhattan.FeedbackSignalManhattanClient +import com.twitter.notificationservice.scribe.manhattan.GenericNotificationsFeedbackRequest +import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse +import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest +import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey +import com.twitter.notificationservice.thriftscala.NotificationService$FinagleClient +import com.twitter.nrel.heavyranker.CandidateFeatureHydrator +import com.twitter.nrel.heavyranker.FeatureHydrator +import com.twitter.nrel.heavyranker.{PushPredictionServiceStore => RelevancePushPredictionServiceStore} +import com.twitter.nrel.heavyranker.{TargetFeatureHydrator => RelevanceTargetFeatureHydrator} +import com.twitter.nrel.lightranker.MagicRecsServeDataRecordLightRanker +import com.twitter.nrel.lightranker.{Config => LightRankerConfig} +import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment +import com.twitter.periscope.api.thriftscala.AudioSpacesLookupContext +import com.twitter.permissions_storage.thriftscala.AppPermission +import com.twitter.recommendation.interests.discovery.core.config.{DeployConfig => InterestDeployConfig} +import com.twitter.recommendation.interests.discovery.popgeo.deploy.PopGeoInterestProvider +import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph +import com.twitter.recos.user_user_graph.thriftscala.UserUserGraph +import com.twitter.rux.common.strato.thriftscala.UserTargetingProperty +import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWProducer +import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation +import com.twitter.search.earlybird.thriftscala.EarlybirdService +import com.twitter.service.gen.scarecrow.thriftscala.ScarecrowService +import com.twitter.service.metastore.gen.thriftscala.Location +import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities +import com.twitter.socialgraph.thriftscala.SocialGraphService +import com.twitter.spam.rtf.thriftscala.SafetyLevel +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storage.client.manhattan.kv.Guarantee +import com.twitter.storage.client.manhattan.kv.ManhattanKVClient +import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams +import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint +import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder +import com.twitter.storehaus.ReadableStore +import com.twitter.storehaus_internal.manhattan.Apollo +import com.twitter.storehaus_internal.manhattan.Athena +import com.twitter.storehaus_internal.manhattan.Dataset +import com.twitter.storehaus_internal.manhattan.ManhattanStore +import com.twitter.storehaus_internal.manhattan.Nash +import com.twitter.storehaus_internal.manhattan.Omega +import com.twitter.storehaus_internal.memcache.MemcacheStore +import com.twitter.storehaus_internal.util.ClientName +import com.twitter.storehaus_internal.util.ZkEndPoint +import com.twitter.strato.catalog.Scan.Slice +import com.twitter.strato.client.Strato +import com.twitter.strato.client.UserId +import com.twitter.strato.columns.frigate.logged_out_web_notifications.thriftscala.LOWebNotificationMetadata +import com.twitter.strato.columns.notifications.thriftscala.SourceDestUserRequest +import com.twitter.strato.generated.client.geo.user.FrequentSoftUserLocationClientColumn +import com.twitter.strato.generated.client.ml.featureStore.TimelineScorerTweetScoresV1ClientColumn +import com.twitter.strato.generated.client.notifications.space_device_follow_impl.SpaceDeviceFollowingClientColumn +import com.twitter.strato.generated.client.periscope.CoreOnAudioSpaceClientColumn +import com.twitter.strato.generated.client.periscope.ParticipantsOnAudioSpaceClientColumn +import com.twitter.strato.generated.client.rux.TargetingPropertyOnUserClientColumn +import com.twitter.strato.generated.client.socialgraph.graphs.creatorSubscriptionTimeline.{CountEdgesBySourceClientColumn => CreatorSubscriptionNumTweetsColumn} +import com.twitter.strato.generated.client.translation.service.IsTweetTranslatableClientColumn +import com.twitter.strato.generated.client.translation.service.platform.MachineTranslateTweetClientColumn +import com.twitter.strato.generated.client.trends.trip.TripTweetsAirflowProdClientColumn +import com.twitter.strato.thrift.ScroogeConvImplicits._ +import com.twitter.taxi.common.AppId +import com.twitter.taxi.deploy.Cluster +import com.twitter.taxi.deploy.Env +import com.twitter.topiclisting.TopicListing +import com.twitter.topiclisting.TopicListingBuilder +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets +import com.twitter.tsp.thriftscala.TopicSocialProofRequest +import com.twitter.tsp.thriftscala.TopicSocialProofResponse +import com.twitter.tweetypie.thriftscala.GetTweetOptions +import com.twitter.tweetypie.thriftscala.Tweet.VisibleTextRangeField +import com.twitter.tweetypie.thriftscala.TweetService +import com.twitter.ubs.thriftscala.AudioSpace +import com.twitter.ubs.thriftscala.Participants +import com.twitter.ubs.thriftscala.SellerApplicationState +import com.twitter.user_session_store.thriftscala.UserSession +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.util.Timer +import com.twitter.util.tunable.TunableMap +import com.twitter.wtf.scalding.common.thriftscala.UserFeatures +import org.apache.thrift.protocol.TCompactProtocol +import com.twitter.timelinescorer.thriftscala.v1.ScoredTweet +import com.twitter.ubs.thriftscala.SellerTrack +import com.twitter.wtf.candidate.thriftscala.CandidateSeq + +trait DeployConfig extends Config { + // Any finagle clients should not be defined as lazy. If defined lazy, + // ClientRegistry.expAllRegisteredClientsResolved() call in init will not ensure that the clients + // are active before thrift endpoint is active. We want the clients to be active, because zookeeper + // resolution triggered by first request(s) might result in the request(s) failing. + + def serviceIdentifier: ServiceIdentifier + + def tunableMap: TunableMap + + def featureSwitches: FeatureSwitches + + override val isProd: Boolean = + serviceIdentifier.environment == PushConstants.ServiceProdEnvironmentName + + def shardParams: ShardParams + + def log: Logger + + implicit def statsReceiver: StatsReceiver + + implicit val timer: Timer = DefaultTimer + + def notifierThriftClientId: ClientId + + def loggedOutNotifierThriftClientId: ClientId + + def pushserviceThriftClientId: ClientId + + def deepbirdv2PredictionServiceDest: String + + def featureStoreUtil: FeatureStoreUtil + + def targetLevelFeaturesConfig: PushFeaturesConfig + + private val manhattanClientMtlsParams = ManhattanKVClientMtlsParams( + serviceIdentifier = serviceIdentifier, + opportunisticTls = OpportunisticTls.Required + ) + + // Commonly used clients + val gizmoduckClient = { + + val client = ThriftMux.client + .withMutualTls(serviceIdentifier) + .withClientId(pushserviceThriftClientId) + .build[UserService.MethodPerEndpoint]( + dest = "/s/gizmoduck/gizmoduck" + ) + + /** + * RequestContext test user config to allow reading test user accounts on pushservice for load + * testing + */ + val GizmoduckTestUserConfig = TestUserConfig( + clientId = Some(pushserviceThriftClientId.name), + readConfig = Some(ReadConfig(includeTestUsers = true)) + ) + + TestUserClientBuilder[UserService.MethodPerEndpoint] + .withClient(client) + .withConfig(GizmoduckTestUserConfig) + .build() + } + + val sgsClient = { + val service = readOnlyThriftService( + "", + "/s/socialgraph/socialgraph", + statsReceiver, + pushserviceThriftClientId, + mTLSServiceIdentifier = Some(serviceIdentifier) + ) + new SocialGraphService.FinagledClient(service) + } + + val tweetyPieClient = { + val service = readOnlyThriftService( + "", + "/s/tweetypie/tweetypie", + statsReceiver, + notifierThriftClientId, + mTLSServiceIdentifier = Some(serviceIdentifier) + ) + new TweetService.FinagledClient(service) + } + + lazy val geoduckHydrationClient: Hydration.MethodPerEndpoint = { + val servicePerEndpoint = ThriftMux.client + .withLabel("geoduck_hydration") + .withClientId(pushserviceThriftClientId) + .withMutualTls(serviceIdentifier) + .methodBuilder("/s/geo/hydration") + .withTimeoutPerRequest(10.seconds) + .withTimeoutTotal(10.seconds) + .idempotent(maxExtraLoad = 0.0) + .servicePerEndpoint[Hydration.ServicePerEndpoint] + Hydration.MethodPerEndpoint(servicePerEndpoint) + } + + lazy val geoduckLocationClient: LocationService.MethodPerEndpoint = { + val servicePerEndpoint = ThriftMux.client + .withLabel("geoduck_location") + .withClientId(pushserviceThriftClientId) + .withMutualTls(serviceIdentifier) + .methodBuilder("/s/geo/geoduck_locationservice") + .withTimeoutPerRequest(10.seconds) + .withTimeoutTotal(10.seconds) + .idempotent(maxExtraLoad = 0.0) + .servicePerEndpoint[LocationService.ServicePerEndpoint] + LocationService.MethodPerEndpoint(servicePerEndpoint) + } + + override val geoDuckV2Store: ReadableStore[Long, LocationResponse] = { + val geoduckLocate: GeoduckUserLocate = GeoduckUserLocateModule.providesGeoduckUserLocate( + locationServiceClient = geoduckLocationClient, + hydrationClient = geoduckHydrationClient, + unscopedStatsReceiver = statsReceiver + ) + + val store: ReadableStore[Long, LocationResponse] = ReadableStore + .convert[GeoduckRequest, Long, LocationResponse, LocationResponse]( + GeoduckStoreV2(geoduckLocate))({ userId: Long => + GeoduckRequest( + userId, + placeTypes = Set( + PlaceType.City, + PlaceType.Metro, + PlaceType.Country, + PlaceType.ZipCode, + PlaceType.Admin0, + PlaceType.Admin1), + placeFields = Set(PlaceQueryFields.PlaceNames), + includeCountryCode = true + ) + })({ locationResponse: LocationResponse => Future.value(locationResponse) }) + + val _cacheName = "geoduckv2_in_memory_cache" + ObservedCachedReadableStore.from( + store, + ttl = 20.seconds, + maxKeys = 1000, + cacheName = _cacheName, + windowSize = 10000L + )(statsReceiver.scope(_cacheName)) + } + + private val deepbirdServiceBase = ThriftMux.client + .withClientId(pushserviceThriftClientId) + .withMutualTls(serviceIdentifier) + .withLoadBalancer(Balancers.p2c()) + .newService(deepbirdv2PredictionServiceDest, "DeepbirdV2PredictionService") + val deepbirdPredictionServiceClient = new DeepbirdPredictionService.ServiceToClient( + Finagle + .retryReadFilter( + tries = 3, + statsReceiver = statsReceiver.scope("DeepbirdV2PredictionService")) + .andThen(Finagle.timeoutFilter(timeout = 10.seconds)) + .andThen(deepbirdServiceBase), + RichClientParam(serviceName = "DeepbirdV2PredictionService", clientStats = statsReceiver) + ) + + val manhattanStarbuckAppId = "frigate_pushservice_starbuck" + val metastoreLocationAppId = "frigate_notifier_metastore_location" + val manhattanMetastoreAppId = "frigate_pushservice_penguin" + + def pushServiceMHCacheDest: String + def pushServiceCoreSvcsCacheDest: String + def poptartImpressionsCacheDest: String = "/srv#/prod/local/cache/poptart_impressions" + def entityGraphCacheDest: String + + val pushServiceCacheClient: Client = MemcacheStore.memcachedClient( + name = ClientName("memcache-pushservice"), + dest = ZkEndPoint(pushServiceMHCacheDest), + statsReceiver = statsReceiver, + timeout = 2.seconds, + serviceIdentifier = serviceIdentifier + ) + + val pushServiceCoreSvcsCacheClient: Client = + MemcacheStore.memcachedClient( + name = ClientName("memcache-pushservice-core-svcs"), + dest = ZkEndPoint(pushServiceCoreSvcsCacheDest), + statsReceiver = statsReceiver, + serviceIdentifier = serviceIdentifier, + timeout = 2.seconds, + ) + + val poptartImpressionsCacheClient: Client = + MemcacheStore.memcachedClient( + name = ClientName("memcache-pushservice-poptart-impressions"), + dest = ZkEndPoint(poptartImpressionsCacheDest), + statsReceiver = statsReceiver, + serviceIdentifier = serviceIdentifier, + timeout = 2.seconds + ) + + val entityGraphCacheClient: Client = MemcacheStore.memcachedClient( + name = ClientName("memcache-pushservice-entity-graph"), + dest = ZkEndPoint(entityGraphCacheDest), + statsReceiver = statsReceiver, + serviceIdentifier = serviceIdentifier, + timeout = 2.seconds + ) + + val stratoClient = { + val pushserviceThriftClient = ThriftMux.client.withClientId(pushserviceThriftClientId) + val baseBuilder = Strato + .Client(pushserviceThriftClient) + .withMutualTls(serviceIdentifier) + val finalBuilder = if (isServiceLocal) { + baseBuilder.withRequestTimeout(Duration.fromSeconds(15)) + } else { + baseBuilder.withRequestTimeout(Duration.fromSeconds(3)) + } + finalBuilder.build() + } + + val interestThriftServiceClient = ThriftMux.client + .withClientId(pushserviceThriftClientId) + .withMutualTls(serviceIdentifier) + .withRequestTimeout(3.seconds) + .configured(Retries.Policy(RetryPolicy.tries(1))) + .configured(BackupRequestFilter.Configured(maxExtraLoad = 0.0, sendInterrupts = false)) + .withStatsReceiver(statsReceiver) + .build[InterestsThriftService.MethodPerEndpoint]( + dest = "/s/interests-thrift-service/interests-thrift-service", + label = "interests-lookup" + ) + + def memcacheCASDest: String + + override val casLock: CasLock = { + val magicrecsCasMemcacheClient = Memcached.client + .withMutualTls(serviceIdentifier) + .withLabel("mr-cas-memcache-client") + .withRequestTimeout(3.seconds) + .withStatsReceiver(statsReceiver) + .configured(Retries.Policy(RetryPolicy.tries(3))) + .newTwemcacheClient(memcacheCASDest) + .withStrings + + MemcacheCasLock(magicrecsCasMemcacheClient) + } + + override val pushInfoStore: ReadableStore[Long, UserForPushTargeting] = { + StratoFetchableStore.withUnitView[Long, UserForPushTargeting]( + stratoClient, + "frigate/magicrecs/pushRecsTargeting.User") + } + + override val loggedOutPushInfoStore: ReadableStore[Long, LOWebNotificationMetadata] = { + StratoFetchableStore.withUnitView[Long, LOWebNotificationMetadata]( + stratoClient, + "frigate/magicrecs/web/loggedOutWebUserStoreMh" + ) + } + + // Setting up model stores + override val dauProbabilityStore: ReadableStore[Long, DauProbability] = { + StratoFetchableStore + .withUnitView[Long, DauProbability](stratoClient, "frigate/magicrecs/dauProbability.User") + } + + override val nsfwConsumerStore = { + StratoFetchableStore.withUnitView[Long, NSFWUserSegmentation]( + stratoClient, + "frigate/nsfw-user-segmentation/nsfwUserSegmentation.User") + } + + override val nsfwProducerStore = { + StratoFetchableStore.withUnitView[Long, NSFWProducer]( + stratoClient, + "frigate/nsfw-user-segmentation/nsfwProducer.User" + ) + } + + override val idsStore: ReadableStore[RecommendedListsRequest, RecommendedListsResponse] = { + val service = Finagle.readOnlyThriftService( + name = "interests-discovery-service", + dest = "/s/interests_discovery/interests_discovery", + statsReceiver, + pushserviceThriftClientId, + requestTimeout = 4.seconds, + tries = 2, + mTLSServiceIdentifier = Some(serviceIdentifier) + ) + val client = new InterestsDiscoveryService.FinagledClient( + service = service, + RichClientParam(serviceName = "interests-discovery-service") + ) + + InterestDiscoveryStore(client) + } + + override val popGeoLists = { + StratoFetchableStore.withUnitView[String, NonPersonalizedRecommendedLists]( + stratoClient, + column = "recommendations/interests_discovery/recommendations_mh/OrganicPopgeoLists" + ) + } + + override val listAPIStore = { + val fetcher = stratoClient + .fetcher[Long, ApiListView, ApiList]("channels/hydration/apiList.List") + StratoFetchableStore.withView[Long, ApiListView, ApiList]( + fetcher, + ApiListView(ApiListDisplayLocation.Recommendations) + ) + } + + override val reactivatedUserInfoStore = { + val stratoFetchableStore = StratoFetchableStore + .withUnitView[Long, String](stratoClient, "ml/featureStore/recentReactivationTime.User") + + ObservedReadableStore( + stratoFetchableStore + )(statsReceiver.scope("RecentReactivationTime")) + } + + override val openedPushByHourAggregatedStore: ReadableStore[Long, Map[Int, Int]] = { + StratoFetchableStore + .withUnitView[Long, Map[Int, Int]]( + stratoClient, + "frigate/magicrecs/opendPushByHourAggregated.User") + } + + private val lexClient: LiveVideoTimelineClient = { + val lexService = + new TimelineService.FinagledClient( + readOnlyThriftService( + name = "lex", + dest = lexServiceDest, + statsReceiver = statsReceiver.scope("lex-service"), + thriftClientId = pushserviceThriftClientId, + requestTimeout = 5.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ), + clientParam = RichClientParam(serviceName = "lex") + ) + new LiveVideoTimelineClient(lexService) + } + + override val lexServiceStore = { + ObservedCachedReadableStore.from[EventRequest, LiveEvent]( + buildStore(LexServiceStore(lexClient), "lexServiceStore"), + ttl = 1.hour, + maxKeys = 1000, + cacheName = "lexServiceStore_cache", + windowSize = 10000L + )(statsReceiver.scope("lexServiceStore_cache")) + } + + val inferredEntitiesFromInterestedInKeyedByClusterColumn = + "recommendations/simclusters_v2/inferred_entities/inferredEntitiesFromInterestedInKeyedByCluster" + override val simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities] = { + val store = StratoFetchableStore + .withUnitView[Int, SimClustersInferredEntities]( + stratoClient, + inferredEntitiesFromInterestedInKeyedByClusterColumn) + ObservedCachedReadableStore.from[Int, SimClustersInferredEntities]( + buildStore(store, "simcluster_entity_store_cache"), + ttl = 6.hours, + maxKeys = 1000, + cacheName = "simcluster_entity_store_cache", + windowSize = 10000L + )(statsReceiver.scope("simcluster_entity_store_cache")) + } + + def fanoutMetadataColumn: String + + override val fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent] = { + val store = StratoFetchableStore + .withUnitView[(Long, Long), FanoutEvent](stratoClient, fanoutMetadataColumn) + ObservedCachedReadableStore.from[(Long, Long), FanoutEvent]( + buildStore(store, "fanoutMetadataStore"), + ttl = 10.minutes, + maxKeys = 1000, + cacheName = "fanoutMetadataStore_cache", + windowSize = 10000L + )(statsReceiver.scope("fanoutMetadataStore_cache")) + } + + /** + * PostRanking Feature Store Client + */ + override def postRankingFeatureStoreClient = { + val clientStats = statsReceiver.scope("post_ranking_feature_store_client") + val clientConfig = + FeatureStoreClientBuilder.getClientConfig(PostRankingFeaturesConfig(), featureStoreUtil) + + FeatureStoreClientBuilder.getDynamicFeatureStoreClient(clientConfig, clientStats) + } + + /** + * Interests lookup store + */ + override val interestsWithLookupContextStore = { + ObservedCachedReadableStore.from[InterestsLookupRequestWithContext, Interests]( + buildStore( + new InterestsWithLookupContextStore(interestThriftServiceClient, statsReceiver), + "InterestsWithLookupContextStore" + ), + ttl = 1.minute, + maxKeys = 1000, + cacheName = "interestsWithLookupContextStore_cache", + windowSize = 10000L + ) + } + + /** + * OptOutInterestsStore + */ + override lazy val optOutUserInterestsStore: ReadableStore[Long, Seq[InterestId]] = { + buildStore( + InterestsOptOutwithLookUpContextStore(interestThriftServiceClient), + "InterestsOptOutStore" + ) + } + + override val topicListing: TopicListing = + if (isServiceLocal) { + new TopicListingBuilder(statsReceiver.scope("topiclisting"), Some(localConfigRepoPath)).build + } else { + new TopicListingBuilder(statsReceiver.scope("topiclisting"), None).build + } + + val cachedUttClient = { + val DefaultUttCacheConfig = CacheConfigV2(capacity = 100) + val uttClientCacheConfigs = uttclient.UttClientCacheConfigsV2( + DefaultUttCacheConfig, + DefaultUttCacheConfig, + DefaultUttCacheConfig, + DefaultUttCacheConfig + ) + new CachedUttClientV2(stratoClient, Environment.Prod, uttClientCacheConfigs, statsReceiver) + } + + override val uttEntityHydrationStore = + new UttEntityHydrationStore(cachedUttClient, statsReceiver, log) + + private lazy val dbv2PredictionServiceScoreStore: RelevancePushPredictionServiceStore = + DeepbirdV2ModelConfig.buildPredictionServiceScoreStore( + deepbirdPredictionServiceClient, + "deepbirdv2_magicrecs" + ) + + // Customized model to PredictionServiceStoreMap + // It is used to specify the predictionServiceStore for the models not in the default dbv2PredictionServiceScoreStore + private lazy val modelToPredictionServiceStoreMap: Map[ + WeightedOpenOrNtabClickModel.ModelNameType, + RelevancePushPredictionServiceStore + ] = Map() + + override lazy val weightedOpenOrNtabClickModelScorer = new PushMLModelScorer( + PushMLModel.WeightedOpenOrNtabClickProbability, + modelToPredictionServiceStoreMap, + dbv2PredictionServiceScoreStore, + statsReceiver.scope("weighted_oonc_scoring") + ) + + override lazy val optoutModelScorer = new PushMLModelScorer( + PushMLModel.OptoutProbability, + Map.empty, + dbv2PredictionServiceScoreStore, + statsReceiver.scope("optout_scoring") + ) + + override lazy val filteringModelScorer = new PushMLModelScorer( + PushMLModel.FilteringProbability, + Map.empty, + dbv2PredictionServiceScoreStore, + statsReceiver.scope("filtering_scoring") + ) + + private val queryFields: Set[QueryFields] = Set( + QueryFields.Profile, + QueryFields.Account, + QueryFields.Roles, + QueryFields.Discoverability, + QueryFields.Safety, + QueryFields.Takedowns, + QueryFields.Labels, + QueryFields.Counts, + QueryFields.ExtendedProfile + ) + + // Setting up safeUserStore + override val safeUserStore = + // in-memory cache + ObservedCachedReadableStore.from[Long, User]( + ObservedReadableStore( + GizmoduckUserStore.safeStore( + client = gizmoduckClient, + queryFields = queryFields, + safetyLevel = SafetyLevel.FilterNone, + statsReceiver = statsReceiver + ) + )(statsReceiver.scope("SafeUserStore")), + ttl = 1.minute, + maxKeys = 5e4.toInt, + cacheName = "safeUserStore_cache", + windowSize = 10000L + )(statsReceiver.scope("safeUserStore_cache")) + + val mobileSdkStore = MobileSdkStore( + "frigate_mobile_sdk_version_apollo", + "mobile_sdk_versions_scalding", + manhattanClientMtlsParams, + Apollo + ) + + val deviceUserStore = ObservedReadableStore( + GizmoduckUserStore( + client = gizmoduckClient, + queryFields = Set(QueryFields.Devices), + context = LookupContext(includeSoftUsers = true), + statsReceiver = statsReceiver + ) + )(statsReceiver.scope("devicesUserStore")) + + override val deviceInfoStore = DeviceInfoStore( + ObservedMemcachedReadableStore.fromCacheClient( + backingStore = ObservedReadableStore( + mobileSdkStore + )(statsReceiver.scope("uncachedMobileSdkVersionsStore")), + cacheClient = pushServiceCacheClient, + ttl = 12.hours + )( + valueInjection = BinaryScalaCodec(SdkVersionValue), + statsReceiver = statsReceiver.scope("MobileSdkVersionsStore"), + keyToString = { + case SdkVersionKey(Some(userId), Some(clientId)) => + s"DeviceInfoStore/$userId/$clientId" + case SdkVersionKey(Some(userId), None) => s"DeviceInfoStore/$userId/_" + case SdkVersionKey(None, Some(clientId)) => + s"DeviceInfoStore/_/$clientId" + case SdkVersionKey(None, None) => s"DeviceInfoStore/_" + } + ), + deviceUserStore + ) + + // Setting up edgeStore + override val edgeStore = SocialGraphPredicate.buildEdgeStore(sgsClient) + + override val socialGraphServiceProcessStore = SocialGraphServiceProcessStore(edgeStore) + + def userTweetEntityGraphDest: String + def userUserGraphDest: String + def lexServiceDest: String + + // Setting up the history store + def frigateHistoryCacheDest: String + + val notificationHistoryStore: NotificationHistoryStore = { + + val manhattanStackBasedClient = ThriftMux.client + .withClientId(notifierThriftClientId) + .withOpportunisticTls(OpportunisticTls.Required) + .withMutualTls( + serviceIdentifier + ) + + val manhattanHistoryMethodBuilder = manhattanStackBasedClient + .withLabel("manhattan_history_v2") + .withRequestTimeout(10.seconds) + .withStatsReceiver(statsReceiver) + .methodBuilder(Omega.wilyName) + .withMaxRetries(3) + + NotificationHistoryStore.build( + "frigate_notifier", + "frigate_notifications_v2", + manhattanHistoryMethodBuilder, + maxRetryCount = 3 + ) + } + + val emailNotificationHistoryStore: ReadOnlyHistoryStore = { + val client = ManhattanKVClient( + appId = "frigate_email_history", + dest = "/s/manhattan/omega.native-thrift", + mtlsParams = ManhattanKVClientMtlsParams( + serviceIdentifier = serviceIdentifier, + opportunisticTls = OpportunisticTls.Required + ) + ) + val endpoint = ManhattanKVEndpointBuilder(client) + .defaultGuarantee(Guarantee.SoftDcReadMyWrites) + .statsReceiver(statsReceiver) + .build() + + ReadOnlyHistoryStore(ManhattanKVHistoryStore(endpoint, dataset = "frigate_email_history"))( + statsReceiver) + } + + val manhattanKVLoggedOutHistoryStoreEndpoint: ManhattanKVEndpoint = { + val mhClient = ManhattanKVClient( + "frigate_notification_logged_out_history", + Nash.wilyName, + manhattanClientMtlsParams) + ManhattanKVEndpointBuilder(mhClient) + .defaultGuarantee(Guarantee.SoftDcReadMyWrites) + .defaultMaxTimeout(5.seconds) + .maxRetryCount(3) + .statsReceiver(statsReceiver) + .build() + } + + val manhattanKVNtabHistoryStoreEndpoint: ManhattanKVEndpoint = { + val mhClient = ManhattanKVClient("frigate_ntab", Omega.wilyName, manhattanClientMtlsParams) + ManhattanKVEndpointBuilder(mhClient) + .defaultGuarantee(Guarantee.SoftDcReadMyWrites) + .defaultMaxTimeout(5.seconds) + .maxRetryCount(3) + .statsReceiver(statsReceiver) + .build() + } + + val nTabHistoryStore: ReadableWritableStore[(Long, String), GenericNotificationOverrideKey] = { + ObservedReadableWritableStore( + NTabHistoryStore(manhattanKVNtabHistoryStoreEndpoint, "frigate_ntab_generic_notif_history") + )(statsReceiver.scope("NTabHistoryStore")) + } + + override lazy val ocfFatigueStore: ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] = + new OCFPromptHistoryStore( + manhattanAppId = "frigate_pushservice_ocf_fatigue_store", + dataset = "fatigue_v1", + manhattanClientMtlsParams + ) + + def historyStore: PushServiceHistoryStore + + def emailHistoryStore: PushServiceHistoryStore + + def loggedOutHistoryStore: PushServiceHistoryStore + + override val hydratedLabeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue] = { + val labeledHistoryMemcacheClient = { + MemcacheStore.memcachedClient( + name = ClientName("history-memcache"), + dest = ZkEndPoint(frigateHistoryCacheDest), + statsReceiver = statsReceiver, + timeout = 2.seconds, + serviceIdentifier = serviceIdentifier + ) + } + + implicit val keyCodec = CompactScalaCodec(UserHistoryKey) + implicit val valueCodec = CompactScalaCodec(UserHistoryValue) + val dataset: Dataset[UserHistoryKey, UserHistoryValue] = + Dataset( + "", + "frigate_data_pipeline_pushservice", + "labeled_push_recs_aggregated_hydrated", + Athena + ) + ObservedMemcachedReadableStore.fromCacheClient( + backingStore = ObservedReadableStore(buildManhattanStore(dataset))( + statsReceiver.scope("UncachedHydratedLabeledPushRecsStore") + ), + cacheClient = labeledHistoryMemcacheClient, + ttl = 6.hours + )( + valueInjection = valueCodec, + statsReceiver = statsReceiver.scope("HydratedLabeledPushRecsStore"), + keyToString = { + case UserHistoryKey.UserId(userId) => s"HLPRS/$userId" + case unknownKey => + throw new IllegalArgumentException(s"Unknown userHistoryStore cache key $unknownKey") + } + ) + } + + override val realTimeClientEventStore: RealTimeClientEventStore = { + val client = ManhattanKVClient( + "frigate_eventstream", + "/s/manhattan/omega.native-thrift", + manhattanClientMtlsParams + ) + val endpoint = + ManhattanKVEndpointBuilder(client) + .defaultGuarantee(Guarantee.SoftDcReadMyWrites) + .defaultMaxTimeout(3.seconds) + .statsReceiver(statsReceiver) + .build() + + ManhattanRealTimeClientEventStore(endpoint, "realtime_client_events", statsReceiver, None) + } + + override val onlineUserHistoryStore: ReadableStore[OnlineUserHistoryKey, UserHistoryValue] = { + OnlineUserHistoryStore(realTimeClientEventStore) + } + + override val userMediaRepresentationStore = UserMediaRepresentationStore( + "user_media_representation", + "user_media_representation_dataset", + manhattanClientMtlsParams + ) + + override val producerMediaRepresentationStore = ObservedMemcachedReadableStore.fromCacheClient( + backingStore = UserMediaRepresentationStore( + "user_media_representation", + "producer_media_representation_dataset", + manhattanClientMtlsParams + )(statsReceiver.scope("UncachedProducerMediaRepStore")), + cacheClient = pushServiceCacheClient, + ttl = 4.hours + )( + valueInjection = BinaryScalaCodec(UserMediaRepresentation), + keyToString = { k: Long => s"ProducerMediaRepStore/$k" }, + statsReceiver.scope("ProducerMediaRepStore") + ) + + override val mrUserStatePredictionStore = { + StratoFetchableStore.withUnitView[Long, MRUserHmmState]( + stratoClient, + "frigate/magicrecs/mrUserStatePrediction.User") + } + + override val userHTLLastVisitStore = + UserHTLLastVisitReadableStore( + "pushservice_htl_user_session", + "tls_user_session_store", + statsReceiver.scope("userHTLLastVisitStore"), + manhattanClientMtlsParams + ) + + val crMixerClient: CrMixer.MethodPerEndpoint = new CrMixer.FinagledClient( + readOnlyThriftService( + "cr-mixer", + "/s/cr-mixer/cr-mixer-plus", + statsReceiver, + pushserviceThriftClientId, + requestTimeout = 5.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ), + clientParam = RichClientParam(serviceName = "cr-mixer") + ) + + val crMixerStore = CrMixerTweetStore(crMixerClient)(statsReceiver.scope("CrMixerTweetStore")) + + val contentMixerClient: ContentMixer.MethodPerEndpoint = new ContentMixer.FinagledClient( + readOnlyThriftService( + "content-mixer", + "/s/corgi-shared/content-mixer", + statsReceiver, + pushserviceThriftClientId, + requestTimeout = 5.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ), + clientParam = RichClientParam(serviceName = "content-mixer") + ) + + val exploreRankerClient: ExploreRanker.MethodPerEndpoint = + new ExploreRanker.FinagledClient( + readOnlyThriftService( + "explore-ranker", + "/s/explore-ranker/explore-ranker", + statsReceiver, + pushserviceThriftClientId, + requestTimeout = 5.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ), + clientParam = RichClientParam(serviceName = "explore-ranker") + ) + + val contentMixerStore = { + ObservedReadableStore(ContentMixerStore(contentMixerClient))( + statsReceiver.scope("ContentMixerStore")) + } + + val exploreRankerStore = { + ObservedReadableStore(ExploreRankerStore(exploreRankerClient))( + statsReceiver.scope("ExploreRankerStore") + ) + } + + val gizmoduckUtcOffsetStore = ObservedReadableStore( + GizmoduckUserUtcOffsetStore.fromUserStore(safeUserStore) + )(statsReceiver.scope("GizmoUserUtcOffsetStore")) + + override val userUtcOffsetStore = + UtcOffsetStore + .makeMemcachedUtcOffsetStore( + gizmoduckUtcOffsetStore, + pushServiceCoreSvcsCacheClient, + ReadableStore.empty, + manhattanStarbuckAppId, + manhattanClientMtlsParams + )(statsReceiver) + .mapValues(Duration.fromSeconds) + + override val cachedTweetyPieStoreV2 = { + val getTweetOptions = Some( + GetTweetOptions( + safetyLevel = Some(SafetyLevel.MagicRecsV2), + includeRetweetCount = true, + includeReplyCount = true, + includeFavoriteCount = true, + includeQuotedTweet = true, + additionalFieldIds = Seq(VisibleTextRangeField.id) + ) + ) + buildCachedTweetyPieStore(getTweetOptions, "tp_v2") + } + + override val cachedTweetyPieStoreV2NoVF = { + val getTweetOptions = Some( + GetTweetOptions( + safetyLevel = Some(SafetyLevel.FilterDefault), + includeRetweetCount = true, + includeReplyCount = true, + includeFavoriteCount = true, + includeQuotedTweet = true, + additionalFieldIds = Seq(VisibleTextRangeField.id), + ) + ) + buildCachedTweetyPieStore(getTweetOptions, "tp_v2_noVF") + } + + override val safeCachedTweetyPieStoreV2 = { + val getTweetOptions = Some( + GetTweetOptions( + safetyLevel = Some(SafetyLevel.MagicRecsAggressiveV2), + includeRetweetCount = true, + includeReplyCount = true, + includeFavoriteCount = true, + includeQuotedTweet = true, + additionalFieldIds = Seq(VisibleTextRangeField.id) + ) + ) + buildCachedTweetyPieStore(getTweetOptions, "sftp_v2") + } + + override val userTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult] = { + val getTweetOptions = Some( + GetTweetOptions( + safetyLevel = Some(SafetyLevel.MagicRecsV2), + includeRetweetCount = true, + includeReplyCount = true, + includeFavoriteCount = true, + includeQuotedTweet = true, + additionalFieldIds = Seq(VisibleTextRangeField.id) + ) + ) + TweetyPieStore.buildUserTweetStore( + client = tweetyPieClient, + options = getTweetOptions + ) + } + + override val safeUserTweetTweetyPieStore: ReadableStore[UserTweet, TweetyPieResult] = { + val getTweetOptions = Some( + GetTweetOptions( + safetyLevel = Some(SafetyLevel.MagicRecsAggressiveV2), + includeRetweetCount = true, + includeReplyCount = true, + includeFavoriteCount = true, + includeQuotedTweet = true, + additionalFieldIds = Seq(VisibleTextRangeField.id) + ) + ) + TweetyPieStore.buildUserTweetStore( + client = tweetyPieClient, + options = getTweetOptions + ) + } + + override val tweetContentFeatureCacheStore: ReadableStore[Long, ThriftDataRecord] = { + ObservedMemcachedReadableStore.fromCacheClient( + backingStore = TweetContentFeatureReadableStore(stratoClient), + cacheClient = poptartImpressionsCacheClient, + ttl = 12.hours + )( + valueInjection = BinaryScalaCodec(ThriftDataRecord), + statsReceiver = statsReceiver.scope("TweetContentFeaturesCacheStore"), + keyToString = { k: Long => s"tcf/$k" } + ) + } + + lazy val tweetTranslationStore: ReadableStore[ + TweetTranslationStore.Key, + TweetTranslationStore.Value + ] = { + val isTweetTranslatableStore = + StratoFetchableStore + .withUnitView[IsTweetTranslatableClientColumn.Key, Boolean]( + fetcher = new IsTweetTranslatableClientColumn(stratoClient).fetcher + ) + + val translateTweetStore = + StratoFetchableStore + .withUnitView[MachineTranslateTweetClientColumn.Key, MachineTranslationResponse]( + fetcher = new MachineTranslateTweetClientColumn(stratoClient).fetcher + ) + + ObservedReadableStore( + TweetTranslationStore(translateTweetStore, isTweetTranslatableStore, statsReceiver) + )(statsReceiver.scope("tweetTranslationStore")) + } + + val scarecrowClient = new ScarecrowService.FinagledClient( + readOnlyThriftService( + "", + "/s/abuse/scarecrow", + statsReceiver, + notifierThriftClientId, + requestTimeout = 5.second, + mTLSServiceIdentifier = Some(serviceIdentifier) + ), + clientParam = RichClientParam(serviceName = "") + ) + + // Setting up scarecrow store + override val scarecrowCheckEventStore = { + ScarecrowCheckEventStore(scarecrowClient) + } + + // setting up the perspective store + override val userTweetPerspectiveStore = { + val service = new DynamicRequestMeterFilter( + tunableMap(PushServiceTunableKeys.TweetPerspectiveStoreQpsLimit), + RateLimiterGenerator.asTuple(_, shardParams.numShards, 40), + PushQPSLimitConstants.PerspectiveStoreQPS)(timer) + .andThen( + readOnlyThriftService( + "tweetypie_perspective_service", + "/s/tweetypie/tweetypie", + statsReceiver, + notifierThriftClientId, + mTLSServiceIdentifier = Some(serviceIdentifier) + ) + ) + + val client = new TweetService.FinagledClient( + service, + clientParam = RichClientParam(serviceName = "tweetypie_perspective_client")) + ObservedReadableStore( + PerspectiveReadableStore(client) + )(statsReceiver.scope("TweetPerspectiveStore")) + } + + //user country code store, used in RecsWithheldContentPredicate - wrapped by memcache based cache + override val userCountryStore = + ObservedMemcachedReadableStore.fromCacheClient( + backingStore = ObservedReadableStore( + UserCountryStore(metastoreLocationAppId, manhattanClientMtlsParams) + )(statsReceiver.scope("userCountryStore")), + cacheClient = pushServiceCacheClient, + ttl = 12.hours + )( + valueInjection = BinaryScalaCodec(Location), + statsReceiver = statsReceiver.scope("UserCountryStore"), + keyToString = { k: Long => s"UserCountryStore/$k" } + ) + + override val audioSpaceParticipantsStore: ReadableStore[String, Participants] = { + val store = StratoFetchableStore + .DefaultStratoFetchableStore( + fetcher = new ParticipantsOnAudioSpaceClientColumn(stratoClient).fetcher + ).composeKeyMapping[String](broadcastId => + (broadcastId, AudioSpacesLookupContext(forUserId = None))) + + ObservedCachedReadableStore + .from( + store = buildStore(store, "AudioSpaceParticipantsStore"), + ttl = 20.seconds, + maxKeys = 200, + cacheName = "AudioSpaceParticipantsStore", + windowSize = 200 + ) + + } + + override val topicSocialProofServiceStore: ReadableStore[ + TopicSocialProofRequest, + TopicSocialProofResponse + ] = { + StratoFetchableStore.withUnitView[TopicSocialProofRequest, TopicSocialProofResponse]( + stratoClient, + "topic-signals/tsp/topic-social-proof") + } + + override val spaceDeviceFollowStore: ReadableStore[SourceDestUserRequest, Boolean] = { + StratoFetchableStore.withUnitView( + fetcher = new SpaceDeviceFollowingClientColumn(stratoClient).fetcher + ) + } + + override val audioSpaceStore: ReadableStore[String, AudioSpace] = { + val store = StratoFetchableStore + .DefaultStratoFetchableStore( + fetcher = new CoreOnAudioSpaceClientColumn(stratoClient).fetcher + ).composeKeyMapping[String] { broadcastId => + (broadcastId, AudioSpacesLookupContext(forUserId = None)) + } + + ObservedCachedReadableStore + .from( + store = buildStore(store, "AudioSpaceVisibilityStore"), + ttl = 1.minute, + maxKeys = 5000, + cacheName = "AudioSpaceVisibilityStore", + windowSize = 10000L) + } + + override val userLanguagesStore = UserLanguagesStore( + manhattanMetastoreAppId, + manhattanClientMtlsParams, + statsReceiver.scope("user_languages_store") + ) + + val tflockClient: TFlockClient = new TFlockClient( + new FlockDB.FinagledClient( + readOnlyThriftService( + "tflockClient", + "/s/tflock/tflock", + statsReceiver, + pushserviceThriftClientId, + mTLSServiceIdentifier = Some(serviceIdentifier) + ), + serviceName = "tflock", + stats = statsReceiver + ), + defaultPageSize = 1000 + ) + + val rawFlockClient = ThriftMux.client + .withClientId(pushserviceThriftClientId) + .withMutualTls(serviceIdentifier) + .build[FlockDB.MethodPerEndpoint]("/s/flock/flock") + + val flockClient: FlockClient = new FlockClient( + rawFlockClient, + defaultPageSize = 100 + ) + + override val recentFollowsStore: FlockFollowStore = { + val dStats = statsReceiver.scope("FlockRecentFollowsStore") + FlockFollowStore(flockClient, dStats) + } + + def notificationServiceClient: NotificationService$FinagleClient + + def notificationServiceSend( + target: Target, + request: CreateGenericNotificationRequest + ): Future[CreateGenericNotificationResponse] + + def notificationServiceDelete( + request: DeleteGenericNotificationRequest + ): Future[Unit] + + def notificationServiceDeleteTimeline( + request: DeleteCurrentTimelineForUserRequest + ): Future[Unit] + + override val notificationServiceSender: ReadableStore[ + NotificationServiceRequest, + CreateGenericNotificationResponse + ] = { + new NotificationServiceSender( + notificationServiceSend, + PushParams.EnableWritesToNotificationServiceParam, + PushParams.EnableWritesToNotificationServiceForAllEmployeesParam, + PushParams.EnableWritesToNotificationServiceForEveryoneParam + ) + } + + val eventRecosServiceClient = { + val dest = "/s/events-recos/events-recos-service" + new EventsRecosService.FinagledClient( + readOnlyThriftService( + "EventRecosService", + dest, + statsReceiver, + pushserviceThriftClientId, + mTLSServiceIdentifier = Some(serviceIdentifier) + ), + clientParam = RichClientParam(serviceName = "EventRecosService") + ) + } + + lazy val recommendedTrendsCandidateSource = RecommendedTrendsCandidateSource( + TrendsRecommendationStore(eventRecosServiceClient, statsReceiver)) + + override val softUserGeoLocationStore: ReadableStore[Long, GeoLocation] = + StratoFetchableStore.withUnitView[Long, GeoLocation](fetcher = + new FrequentSoftUserLocationClientColumn(stratoClient).fetcher) + + lazy val candidateSourceGenerator = new PushCandidateSourceGenerator( + earlybirdCandidateSource, + userTweetEntityGraphCandidates, + cachedTweetyPieStoreV2, + safeCachedTweetyPieStoreV2, + userTweetTweetyPieStore, + safeUserTweetTweetyPieStore, + cachedTweetyPieStoreV2NoVF, + edgeStore, + interestsWithLookupContextStore, + uttEntityHydrationStore, + geoDuckV2Store, + topTweetsByGeoStore, + topTweetsByGeoV2VersionedStore, + ruxTweetImpressionsStore, + recommendedTrendsCandidateSource, + recentTweetsByAuthorsStore, + topicSocialProofServiceStore, + crMixerStore, + contentMixerStore, + exploreRankerStore, + softUserGeoLocationStore, + tripTweetCandidateStore, + popGeoLists, + idsStore + ) + + lazy val loCandidateSourceGenerator = new LoggedOutPushCandidateSourceGenerator( + tripTweetCandidateStore, + geoDuckV2Store, + safeCachedTweetyPieStoreV2, + cachedTweetyPieStoreV2NoVF, + cachedTweetyPieStoreV2, + contentMixerStore, + softUserGeoLocationStore, + topTweetsByGeoStore, + topTweetsByGeoV2VersionedStore + ) + + lazy val rfphStatsRecorder = new RFPHStatsRecorder() + + lazy val rfphRestrictStep = new RFPHRestrictStep() + + lazy val rfphTakeStepUtil = new RFPHTakeStepUtil()(statsReceiver) + + lazy val rfphPrerankFilter = new RFPHPrerankFilter()(statsReceiver) + + lazy val rfphLightRanker = new RFPHLightRanker(lightRanker, statsReceiver) + + lazy val sendHandlerPredicateUtil = new SendHandlerPredicateUtil()(statsReceiver) + + lazy val ntabSender = + new NtabSender( + notificationServiceSender, + nTabHistoryStore, + notificationServiceDelete, + notificationServiceDeleteTimeline + ) + + lazy val ibis2Sender = new Ibis2Sender(pushIbisV2Store, tweetTranslationStore, statsReceiver) + + lazy val historyWriter = new HistoryWriter(historyStore, statsReceiver) + + lazy val loggedOutHistoryWriter = new HistoryWriter(loggedOutHistoryStore, statsReceiver) + + lazy val eventBusWriter = new EventBusWriter(pushSendEventBusPublisher, statsReceiver) + + lazy val ntabOnlyChannelSelector = new NtabOnlyChannelSelector + + lazy val notificationSender = + new NotificationSender( + ibis2Sender, + ntabSender, + statsReceiver, + notificationScribe + ) + + lazy val candidateNotifier = + new CandidateNotifier( + notificationSender, + casLock = casLock, + historyWriter = historyWriter, + eventBusWriter = eventBusWriter, + ntabOnlyChannelSelector = ntabOnlyChannelSelector + )(statsReceiver) + + lazy val loggedOutCandidateNotifier = new CandidateNotifier( + notificationSender, + casLock = casLock, + historyWriter = loggedOutHistoryWriter, + eventBusWriter = null, + ntabOnlyChannelSelector = ntabOnlyChannelSelector + )(statsReceiver) + + lazy val rfphNotifier = + new RefreshForPushNotifier(rfphStatsRecorder, candidateNotifier)(statsReceiver) + + lazy val loRfphNotifier = + new LoggedOutRefreshForPushNotifier(rfphStatsRecorder, loggedOutCandidateNotifier)( + statsReceiver) + + lazy val rfphRanker = { + val randomRanker = RandomRanker[Target, PushCandidate]() + val subscriptionCreatorRanker = + new SubscriptionCreatorRanker(superFollowEligibilityUserStore, statsReceiver) + new RFPHRanker( + randomRanker, + weightedOpenOrNtabClickModelScorer, + subscriptionCreatorRanker, + userHealthSignalStore, + producerMediaRepresentationStore, + statsReceiver + ) + } + + lazy val rfphFeatureHydrator = new RFPHFeatureHydrator(featureHydrator) + lazy val loggedOutRFPHRanker = new LoggedOutRanker(cachedTweetyPieStoreV2, statsReceiver) + + override val userFeaturesStore: ReadableStore[Long, UserFeatures] = { + implicit val valueCodec = new BinaryScalaCodec(UserFeatures) + val dataset: Dataset[Long, UserFeatures] = + Dataset( + "", + "user_features_pushservice_apollo", + "recommendations_user_features_apollo", + Apollo) + + ObservedMemcachedReadableStore.fromCacheClient( + backingStore = ObservedReadableStore(buildManhattanStore(dataset))( + statsReceiver.scope("UncachedUserFeaturesStore") + ), + cacheClient = pushServiceCacheClient, + ttl = 24.hours + )( + valueInjection = valueCodec, + statsReceiver = statsReceiver.scope("UserFeaturesStore"), + keyToString = { k: Long => s"ufts/$k" } + ) + } + + override def htlScoreStore(userId: Long): ReadableStore[Long, ScoredTweet] = { + val fetcher = new TimelineScorerTweetScoresV1ClientColumn(stratoClient).fetcher + val htlStore = buildStore( + StratoFetchableStore.withView[Long, TimelineScorerScoreView, ScoredTweet]( + fetcher, + TimelineScorerScoreView(Some(userId)) + ), + "htlScoreStore" + ) + htlStore + } + + override val userTargetingPropertyStore: ReadableStore[Long, UserTargetingProperty] = { + val name = "userTargetingPropertyStore" + val store = StratoFetchableStore + .withUnitView(new TargetingPropertyOnUserClientColumn(stratoClient).fetcher) + buildStore(store, name) + } + + override val timelinesUserSessionStore: ReadableStore[Long, UserSession] = { + implicit val valueCodec = new CompactScalaCodec(UserSession) + val dataset: Dataset[Long, UserSession] = Dataset[Long, UserSession]( + "", + "frigate_realgraph", + "real_graph_user_features", + Apollo + ) + + ObservedMemcachedReadableStore.fromCacheClient( + backingStore = ObservedReadableStore(buildManhattanStore(dataset))( + statsReceiver.scope("UncachedTimelinesUserSessionStore") + ), + cacheClient = pushServiceCacheClient, + ttl = 6.hours + )( + valueInjection = valueCodec, + statsReceiver = statsReceiver.scope("timelinesUserSessionStore"), + keyToString = { k: Long => s"tluss/$k" } + ) + } + + lazy val recentTweetsFromTflockStore: ReadableStore[Long, Seq[Long]] = + ObservedReadableStore( + RecentTweetsByAuthorsStore.usingRecentTweetsConfig( + tflockClient, + RecentTweetsConfig(maxResults = 1, maxAge = 3.days) + ) + )(statsReceiver.scope("RecentTweetsFromTflockStore")) + + lazy val recentTweetsByAuthorsStore: ReadableStore[RecentTweetsQuery, Seq[Seq[Long]]] = + ObservedReadableStore( + RecentTweetsByAuthorsStore(tflockClient) + )(statsReceiver.scope("RecentTweetsByAuthorsStore")) + + val jobConfig = PopGeoInterestProvider + .getPopularTweetsJobConfig( + InterestDeployConfig( + AppId("PopularTweetsByInterestProd"), + Cluster.ATLA, + Env.Prod, + serviceIdentifier, + manhattanClientMtlsParams + )) + .withManhattanAppId("frigate_pop_by_geo_tweets") + + override val topTweetsByGeoStore = TopTweetsStore.withMemCache( + jobConfig, + pushServiceCacheClient, + 10.seconds + )(statsReceiver) + + override val topTweetsByGeoV2VersionedStore: ReadableStore[String, PopTweetsInPlace] = { + StratoFetchableStore.withUnitView[String, PopTweetsInPlace]( + stratoClient, + "recommendations/popgeo/popGeoTweetsVersioned") + } + + override lazy val pushcapDynamicPredictionStore: ReadableStore[Long, PushcapUserHistory] = { + StratoFetchableStore.withUnitView[Long, PushcapUserHistory]( + stratoClient, + "frigate/magicrecs/pushcapDynamicPrediction.User") + } + + override val tweetAuthorLocationFeatureBuilder = + UserLocationFeatureBuilder(Some("TweetAuthor")) + .withStats() + + override val tweetAuthorLocationFeatureBuilderById = + UserLocationFeatureBuilderById( + userCountryStore, + tweetAuthorLocationFeatureBuilder + ).withStats() + + override val socialContextActionsFeatureBuilder = + SocialContextActionsFeatureBuilder().withStats() + + override val tweetContentFeatureBuilder = + TweetContentFeatureBuilder(tweetContentFeatureCacheStore).withStats() + + override val tweetAuthorRecentRealGraphFeatureBuilder = + RecentRealGraphFeatureBuilder( + stratoClient, + UserAuthorEntity, + TargetUserEntity, + TweetAuthorEntity, + TweetAuthorRecentRealGraphFeatures(statsReceiver.scope("TweetAuthorRecentRealGraphFeatures")) + ).withStats() + + override val socialContextRecentRealGraphFeatureBuilder = + SocialContextRecentRealGraphFeatureBuilder( + RecentRealGraphFeatureBuilder( + stratoClient, + TargetUserSocialContextEntity, + TargetUserEntity, + SocialContextEntity, + SocialContextRecentRealGraphFeatures( + statsReceiver.scope("SocialContextRecentRealGraphFeatures")) + )(statsReceiver + .scope("SocialContextRecentRealGraphFeatureBuilder").scope("RecentRealGraphFeatureBuilder")) + ).withStats() + + override val tweetSocialProofFeatureBuilder = + TweetSocialProofFeatureBuilder(Some("TargetUser")).withStats() + + override val targetUserFullRealGraphFeatureBuilder = + TargetFullRealGraphFeatureBuilder(Some("TargetUser")).withStats() + + override val postProcessingFeatureBuilder: PostProcessingFeatureBuilder = + PostProcessingFeatureBuilder() + + override val mrOfflineUserCandidateSparseAggregatesFeatureBuilder = + MrOfflineUserCandidateSparseAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats() + + override val mrOfflineUserAggregatesFeatureBuilder = + MrOfflineUserAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats() + + override val mrOfflineUserCandidateAggregatesFeatureBuilder = + MrOfflineUserCandidateAggregatesFeatureBuilder(stratoClient, featureStoreUtil).withStats() + + override val tweetAnnotationsFeatureBuilder = + TweetAnnotationsFeatureBuilder(stratoClient).withStats() + + override val targetUserMediaRepresentationFeatureBuilder = + UserMediaRepresentationFeatureBuilder(userMediaRepresentationStore).withStats() + + override val targetLevelFeatureBuilder = + TargetLevelFeatureBuilder(featureStoreUtil, targetLevelFeaturesConfig).withStats() + + override val candidateLevelFeatureBuilder = + CandidateLevelFeatureBuilder(featureStoreUtil).withStats() + + override lazy val targetFeatureHydrator = RelevanceTargetFeatureHydrator( + targetUserFullRealGraphFeatureBuilder, + postProcessingFeatureBuilder, + targetUserMediaRepresentationFeatureBuilder, + targetLevelFeatureBuilder + ) + + override lazy val featureHydrator = + FeatureHydrator(targetFeatureHydrator, candidateFeatureHydrator) + + val pushServiceLightRankerConfig: LightRankerConfig = new LightRankerConfig( + pushserviceThriftClientId, + serviceIdentifier, + statsReceiver.scope("lightRanker"), + deepbirdv2PredictionServiceDest, + "DeepbirdV2PredictionService" + ) + val lightRanker: MagicRecsServeDataRecordLightRanker = + pushServiceLightRankerConfig.lightRanker + + override val tweetImpressionStore: ReadableStore[Long, Seq[Long]] = { + val name = "htl_impression_store" + val store = buildStore( + HtlTweetImpressionStore.createStoreWithTweetIds( + requestTimeout = 6.seconds, + label = "htl_tweet_impressions", + serviceIdentifier = serviceIdentifier, + statsReceiver = statsReceiver + ), + name + ) + val numTweetsReturned = + statsReceiver.scope(name).stat("num_tweets_returned_per_user") + new TransformedReadableStore(store)((userId: Long, tweetIds: Seq[Long]) => { + numTweetsReturned.add(tweetIds.size) + Future.value(Some(tweetIds)) + }) + } + + val ruxTweetImpressionsStore = new TweetImpressionsStore(stratoClient) + + override val strongTiesStore: ReadableStore[Long, STPResult] = { + implicit val valueCodec = new BinaryScalaCodec(STPResult) + val strongTieScoringDataset: Dataset[Long, STPResult] = + Dataset("", "frigate_stp", "stp_result_rerank", Athena) + buildManhattanStore(strongTieScoringDataset) + } + + override lazy val earlybirdFeatureStore = ObservedReadableStore( + EarlybirdFeatureStore( + clientId = pushserviceThriftClientId.name, + earlybirdSearchStore = earlybirdSearchStore + ) + )(statsReceiver.scope("EarlybirdFeatureStore")) + + override lazy val earlybirdFeatureBuilder = EarlybirdFeatureBuilder(earlybirdFeatureStore) + + override lazy val earlybirdSearchStore = { + val earlybirdClientName: String = "earlybird" + val earlybirdSearchStoreName: String = "EarlybirdSearchStore" + + val earlybirdClient = new EarlybirdService.FinagledClient( + readOnlyThriftService( + earlybirdClientName, + earlybirdSearchDest, + statsReceiver, + pushserviceThriftClientId, + tries = 1, + requestTimeout = 3.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ), + clientParam = RichClientParam(protocolFactory = new TCompactProtocol.Factory) + ) + + ObservedReadableStore( + EarlybirdSearchStore(earlybirdClient)(statsReceiver.scope(earlybirdSearchStoreName)) + )(statsReceiver.scope(earlybirdSearchStoreName)) + } + + override lazy val earlybirdCandidateSource: EarlybirdCandidateSource = EarlybirdCandidateSource( + clientId = pushserviceThriftClientId.name, + earlybirdSearchStore = earlybirdSearchStore + ) + + override val realGraphScoresTop500InStore: RealGraphScoresTop500InStore = { + val stratoRealGraphInStore = + StratoFetchableStore + .withUnitView[Long, CandidateSeq]( + stratoClient, + "frigate/magicrecs/fanoutCoi500pRealGraphV2") + + RealGraphScoresTop500InStore( + ObservedMemcachedReadableStore.fromCacheClient( + backingStore = stratoRealGraphInStore, + cacheClient = entityGraphCacheClient, + ttl = 24.hours + )( + valueInjection = BinaryScalaCodec(CandidateSeq), + statsReceiver = statsReceiver.scope("CachedRealGraphScoresTop500InStore"), + keyToString = { k: Long => s"500p_test/$k" } + ) + ) + } + + override val tweetEntityGraphStore = { + val tweetEntityGraphClient = new UserTweetEntityGraph.FinagledClient( + Finagle.readOnlyThriftService( + "user_tweet_entity_graph", + userTweetEntityGraphDest, + statsReceiver, + pushserviceThriftClientId, + requestTimeout = 5.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ) + ) + ObservedReadableStore( + RecommendedTweetEntitiesStore( + tweetEntityGraphClient, + statsReceiver.scope("RecommendedTweetEntitiesStore") + ) + )(statsReceiver.scope("RecommendedTweetEntitiesStore")) + } + + override val userUserGraphStore = { + val userUserGraphClient = new UserUserGraph.FinagledClient( + Finagle.readOnlyThriftService( + "user_user_graph", + userUserGraphDest, + statsReceiver, + pushserviceThriftClientId, + requestTimeout = 5.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ), + clientParam = RichClientParam(serviceName = "user_user_graph") + ) + ObservedReadableStore( + UserUserGraphStore(userUserGraphClient, statsReceiver.scope("UserUserGraphStore")) + )(statsReceiver.scope("UserUserGraphStore")) + } + + override val ntabCaretFeedbackStore: ReadableStore[GenericNotificationsFeedbackRequest, Seq[ + CaretFeedbackDetails + ]] = { + val client = ManhattanKVClient( + "pushservice_ntab_caret_feedback_omega", + Omega.wilyName, + manhattanClientMtlsParams + ) + val endpoint = ManhattanKVEndpointBuilder(client) + .defaultGuarantee(Guarantee.SoftDcReadMyWrites) + .defaultMaxTimeout(3.seconds) + .maxRetryCount(2) + .statsReceiver(statsReceiver) + .build() + + val feedbackSignalManhattanClient = + FeedbackSignalManhattanClient(endpoint, statsReceiver.scope("FeedbackSignalManhattanClient")) + NtabCaretFeedbackStore(feedbackSignalManhattanClient) + } + + override val genericFeedbackStore: ReadableStore[FeedbackRequest, Seq[ + FeedbackPromptValue + ]] = { + FeedbackStore( + GenericFeedbackStoreBuilder.build( + manhattanKVClientAppId = "frigate_pushservice_ntabfeedback_prompt", + environment = NotifEnvironment.apply(serviceIdentifier.environment), + svcIdentifier = serviceIdentifier, + statsReceiver = statsReceiver + )) + } + + override val genericNotificationFeedbackStore: GenericFeedbackStore = { + + GenericFeedbackStoreBuilder.build( + manhattanKVClientAppId = "frigate_pushservice_ntabfeedback_prompt", + environment = NotifEnvironment.apply(serviceIdentifier.environment), + svcIdentifier = serviceIdentifier, + statsReceiver = statsReceiver + ) + } + + override val earlybirdSearchDest = "/s/earlybird-root-superroot/root-superroot" + + // low latency as compared to default `semanticCoreMetadataClient` + private val lowLatencySemanticCoreMetadataClient: MetadataService.MethodPerEndpoint = + new MetadataService.FinagledClient( + Finagle.readOnlyThriftService( + name = "semantic_core_metadata_service", + dest = "/s/escherbird/metadataservice", + statsReceiver = statsReceiver, + thriftClientId = pushserviceThriftClientId, + tries = 2, // total number of tries. number of retries = tries - 1 + requestTimeout = 2.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ) + ) + + private val semanticCoreMetadataStitchClient = new MetadataStitchClient( + lowLatencySemanticCoreMetadataClient + ) + + override val semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata] = { + val name = "semantic_core_megadata_store_cached" + val store = MetaDataReadableStore.getMegadataReadableStore( + metadataStitchClient = semanticCoreMetadataStitchClient, + typedMetadataDomains = Some(Set(Domains.EventsEntityService)) + ) + ObservedCachedReadableStore + .from( + store = ObservedReadableStore(store)( + statsReceiver + .scope("store") + .scope("semantic_core_megadata_store") + ), + ttl = 1.hour, + maxKeys = 1000, + cacheName = "semantic_core_megadata_cache", + windowSize = 10000L + )(statsReceiver.scope("store", name)) + } + + override val basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate] = { + StratoFetchableStore.withUnitView[QualifiedId, BasketballGameLiveUpdate]( + stratoClient, + "semanticCore/basketballGameScore.Entity") + } + + override val baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate] = { + StratoFetchableStore.withUnitView[QualifiedId, BaseballGameLiveUpdate]( + stratoClient, + "semanticCore/baseballGameScore.Entity") + } + + override val cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate] = { + StratoFetchableStore.withUnitView[QualifiedId, CricketMatchLiveUpdate]( + stratoClient, + "semanticCore/cricketMatchScore.Entity") + } + + override val soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate] = { + ObservedCachedReadableStore + .from( + store = StratoFetchableStore.withUnitView[QualifiedId, SoccerMatchLiveUpdate]( + stratoClient, + "semanticCore/soccerMatchScore.Entity"), + ttl = 10.seconds, + maxKeys = 100, + cacheName = "SoccerMatchCachedStore", + windowSize = 100L + )(statsReceiver.scope("SoccerMatchCachedStore")) + + } + + override val nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate] = { + ObservedCachedReadableStore + .from( + store = StratoFetchableStore.withUnitView[QualifiedId, NflFootballGameLiveUpdate]( + stratoClient, + "semanticCore/nflFootballGameScore.Entity"), + ttl = 10.seconds, + maxKeys = 100, + cacheName = "NFLMatchCachedStore", + windowSize = 100L + )(statsReceiver.scope("NFLMatchCachedStore")) + + } + + override val userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse] = { + val userHealthSignalFetcher = + stratoClient.fetcher[Long, Seq[UserHealthSignal], UserHealthSignalResponse]( + "hss/user_signals/api/healthSignals.User" + ) + + val store = buildStore( + StratoFetchableStore.withView[Long, Seq[UserHealthSignal], UserHealthSignalResponse]( + userHealthSignalFetcher, + Seq( + AgathaRecentAbuseStrikeDouble, + AgathaCalibratedNsfwDouble, + AgathaCseDouble, + NsfwTextUserScoreDouble, + NsfwConsumerScoreDouble)), + "UserHealthSignalFetcher" + ) + if (!inMemCacheOff) { + ObservedCachedReadableStore + .from( + store = ObservedReadableStore(store)( + statsReceiver.scope("store").scope("user_health_model_score_store")), + ttl = 12.hours, + maxKeys = 16777215, + cacheName = "user_health_model_score_store_cache", + windowSize = 10000L + )(statsReceiver.scope("store", "user_health_model_score_store_cached")) + } else { + store + } + } + + override val tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] = { + val tweetHealthScoreFetcher = + stratoClient.fetcher[TweetScoringRequest, Unit, TweetScoringResponse]( + "abuse/detection/tweetHealthModelScore" + ) + + val store = buildStore( + StratoFetchableStore.withUnitView(tweetHealthScoreFetcher), + "TweetHealthScoreFetcher" + ) + + ObservedCachedReadableStore + .from( + store = ObservedReadableStore(store)( + statsReceiver.scope("store").scope("tweet_health_model_score_store")), + ttl = 30.minutes, + maxKeys = 1000, + cacheName = "tweet_health_model_score_store_cache", + windowSize = 10000L + )(statsReceiver.scope("store", "tweet_health_model_score_store_cached")) + } + + override val appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission] = { + val store = StratoFetchableStore + .withUnitView[(Long, (String, String)), AppPermission]( + stratoClient, + "clients/permissionsState") + ObservedCachedReadableStore.from[(Long, (String, String)), AppPermission]( + buildStore(store, "mr_app_permission_store"), + ttl = 30.minutes, + maxKeys = 1000, + cacheName = "mr_app_permission_store_cache", + windowSize = 10000L + )(statsReceiver.scope("mr_app_permission_store_cached")) + } + + def pushSendEventStreamName: String + + override val pushSendEventBusPublisher = EventBusPublisherBuilder() + .clientId("frigate_pushservice") + .streamName(pushSendEventStreamName) + .thriftStruct(NotificationScribe) + .statsReceiver(statsReceiver.scope("push_send_eventbus")) + .build() + + override lazy val candidateFeatureHydrator: CandidateFeatureHydrator = + CandidateFeatureHydrator( + socialContextActionsFeatureBuilder = Some(socialContextActionsFeatureBuilder), + tweetSocialProofFeatureBuilder = Some(tweetSocialProofFeatureBuilder), + earlybirdFeatureBuilder = Some(earlybirdFeatureBuilder), + tweetContentFeatureBuilder = Some(tweetContentFeatureBuilder), + tweetAuthorRecentRealGraphFeatureBuilder = Some(tweetAuthorRecentRealGraphFeatureBuilder), + socialContextRecentRealGraphFeatureBuilder = Some(socialContextRecentRealGraphFeatureBuilder), + tweetAnnotationsFeatureBuilder = Some(tweetAnnotationsFeatureBuilder), + mrOfflineUserCandidateSparseAggregatesFeatureBuilder = + Some(mrOfflineUserCandidateSparseAggregatesFeatureBuilder), + candidateLevelFeatureBuilder = Some(candidateLevelFeatureBuilder) + )(statsReceiver.scope("push_feature_hydrator")) + + private val candidateCopyCross = + new CandidateCopyExpansion(statsReceiver.scope("refresh_handler/cross")) + + override lazy val candidateHydrator: PushCandidateHydrator = + PushCandidateHydrator( + this.socialGraphServiceProcessStore, + safeUserStore, + listAPIStore, + candidateCopyCross)( + statsReceiver.scope("push_candidate_hydrator"), + weightedOpenOrNtabClickModelScorer) + + override lazy val sendHandlerCandidateHydrator: SendHandlerPushCandidateHydrator = + SendHandlerPushCandidateHydrator( + lexServiceStore, + fanoutMetadataStore, + semanticCoreMegadataStore, + safeUserStore, + simClusterToEntityStore, + audioSpaceStore, + interestsWithLookupContextStore, + uttEntityHydrationStore, + superFollowCreatorTweetCountStore + )( + statsReceiver.scope("push_candidate_hydrator"), + weightedOpenOrNtabClickModelScorer + ) + + def mrRequestScriberNode: String + def loggedOutMrRequestScriberNode: String + + override lazy val configParamsBuilder: ConfigParamsBuilder = ConfigParamsBuilder( + config = overridesConfig, + featureContextBuilder = FeatureContextBuilder(featureSwitches), + statsReceiver = statsReceiver + ) + + def buildStore[K, V](store: ReadableStore[K, V], name: String): ReadableStore[K, V] = { + ObservedReadableStore(store)(statsReceiver.scope("store").scope(name)) + } + + def buildManhattanStore[K, V](dataset: Dataset[K, V]): ReadableStore[K, V] = { + val manhattanKVClientParams = ManhattanKVClientMtlsParams( + serviceIdentifier = serviceIdentifier, + opportunisticTls = OpportunisticTls.Required + ) + ManhattanStore + .fromDatasetWithMtls[K, V]( + dataset, + mtlsParams = manhattanKVClientParams, + statsReceiver = statsReceiver.scope(dataset.datasetName)) + } + + def buildCachedTweetyPieStore( + getTweetOptions: Option[GetTweetOptions], + keyPrefix: String + ): ReadableStore[Long, TweetyPieResult] = { + def discardAdditionalMediaInfo(tweetypieResult: TweetyPieResult) = { + val updatedMedia = tweetypieResult.tweet.media.map { mediaSeq => + mediaSeq.map { media => media.copy(additionalMetadata = None, sizes = Nil.toSet) } + } + val updatedTweet = tweetypieResult.tweet.copy(media = updatedMedia) + tweetypieResult.copy(tweet = updatedTweet) + } + + val tweetypieStoreWithoutAdditionalMediaInfo = TweetyPieStore( + tweetyPieClient, + getTweetOptions, + transformTweetypieResult = discardAdditionalMediaInfo + )(statsReceiver.scope("tweetypie_without_additional_media_info")) + + ObservedMemcachedReadableStore.fromCacheClient( + backingStore = tweetypieStoreWithoutAdditionalMediaInfo, + cacheClient = pushServiceCoreSvcsCacheClient, + ttl = 12.hours + )( + valueInjection = TweetyPieResultInjection, + statsReceiver = statsReceiver.scope("TweetyPieStore"), + keyToString = { k: Long => s"$keyPrefix/$k" } + ) + } + + override def init(): Future[Unit] = + ClientRegistry.expAllRegisteredClientsResolved().map { clients => + log.info("Done resolving clients: " + clients.mkString("[", ", ", "]")) + } + + val InlineActionsMhColumn = + "frigate/magicrecs/inlineActionsMh" + + override val inlineActionHistoryStore: ReadableStore[Long, Seq[(Long, String)]] = + StratoScannableStore + .withUnitView[(Long, Slice[Long]), (Long, Long), String](stratoClient, InlineActionsMhColumn) + .composeKeyMapping[Long] { userId => + (userId, Slice[Long](from = None, to = None, limit = None)) + }.mapValues { response => + response.map { + case (key, value) => (key._2, value) + } + } + + override val tripTweetCandidateStore: ReadableStore[TripDomain, TripTweets] = { + StratoFetchableStore + .withUnitView[TripDomain, TripTweets]( + new TripTweetsAirflowProdClientColumn(stratoClient).fetcher) + } + + override val softUserFollowingStore: ReadableStore[User, Seq[Long]] = new SoftUserFollowingStore( + stratoClient) + + override val superFollowEligibilityUserStore: ReadableStore[Long, Boolean] = { + StratoFetchableStore.withUnitView[Long, Boolean]( + stratoClient, + "audiencerewards/audienceRewardsService/getSuperFollowEligibility.User") + } + + override val superFollowCreatorTweetCountStore: ReadableStore[UserId, Int] = { + ObservedCachedReadableStore + .from( + store = StratoFetchableStore + .withUnitView[UserId, Int](new CreatorSubscriptionNumTweetsColumn(stratoClient).fetcher), + ttl = 5.minutes, + maxKeys = 1000, + cacheName = "SuperFollowCreatorTweetCountStore", + windowSize = 10000L + )(statsReceiver.scope("SuperFollowCreatorTweetCountStore")) + + } + + override val hasSuperFollowingRelationshipStore: ReadableStore[ + HasSuperFollowingRelationshipRequest, + Boolean + ] = { + StratoFetchableStore.withUnitView[HasSuperFollowingRelationshipRequest, Boolean]( + stratoClient, + "audiencerewards/superFollows/hasSuperFollowingRelationshipV2") + } + + override val superFollowApplicationStatusStore: ReadableStore[ + (Long, SellerTrack), + SellerApplicationState + ] = { + StratoFetchableStore.withUnitView[(Long, SellerTrack), SellerApplicationState]( + stratoClient, + "periscope/eligibility/applicationStatus") + } + + def historyStoreMemcacheDest: String + + override lazy val recentHistoryCacheClient = { + RecentHistoryCacheClient.build(historyStoreMemcacheDest, serviceIdentifier, statsReceiver) + } + + override val openAppUserStore: ReadableStore[Long, Boolean] = { + buildStore(OpenAppUserStore(stratoClient), "OpenAppUserStore") + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.scala new file mode 100644 index 0000000000..923a785eec --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ExperimentsWithStats.scala @@ -0,0 +1,16 @@ +package com.twitter.frigate.pushservice.config + +import com.twitter.frigate.common.util.Experiments + +object ExperimentsWithStats { + + /** + * Add an experiment here to collect detailed pushservice stats. + * + * ! Important ! + * Keep this set small and remove experiments when you don't need the stats anymore. + */ + final val PushExperiments: Set[String] = Set( + Experiments.MRAndroidInlineActionHoldback.exptName, + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.scala new file mode 100644 index 0000000000..7edc8d46d2 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/ProdConfig.scala @@ -0,0 +1,230 @@ +package com.twitter.frigate.pushservice.config + +import com.twitter.abdecider.LoggingABDecider +import com.twitter.bijection.scrooge.BinaryScalaCodec +import com.twitter.bijection.Base64String +import com.twitter.bijection.Injection +import com.twitter.conversions.DurationOps._ +import com.twitter.decider.Decider +import com.twitter.featureswitches.v2.FeatureSwitches +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.finagle.thrift.RichClientParam +import com.twitter.finagle.util.DefaultTimer +import com.twitter.frigate.common.config.RateLimiterGenerator +import com.twitter.frigate.common.filter.DynamicRequestMeterFilter +import com.twitter.frigate.common.history.ManhattanHistoryStore +import com.twitter.frigate.common.history.InvalidatingAfterWritesPushServiceHistoryStore +import com.twitter.frigate.common.history.ManhattanKVHistoryStore +import com.twitter.frigate.common.history.PushServiceHistoryStore +import com.twitter.frigate.common.history.SimplePushServiceHistoryStore +import com.twitter.frigate.common.util._ +import com.twitter.frigate.data_pipeline.features_common.FeatureStoreUtil +import com.twitter.frigate.data_pipeline.features_common.TargetLevelFeaturesConfig +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.DeciderKey +import com.twitter.frigate.pushservice.params.PushQPSLimitConstants +import com.twitter.frigate.pushservice.params.PushServiceTunableKeys +import com.twitter.frigate.pushservice.params.ShardParams +import com.twitter.frigate.pushservice.store.PushIbis2Store +import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe +import com.twitter.frigate.scribe.thriftscala.NotificationScribe +import com.twitter.ibis2.service.thriftscala.Ibis2Service +import com.twitter.logging.Logger +import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest +import com.twitter.notificationservice.api.thriftscala.NotificationApi +import com.twitter.notificationservice.api.thriftscala.NotificationApi$FinagleClient +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse +import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest +import com.twitter.notificationservice.thriftscala.NotificationService +import com.twitter.notificationservice.thriftscala.NotificationService$FinagleClient +import com.twitter.servo.decider.DeciderGateBuilder +import com.twitter.util.tunable.TunableMap +import com.twitter.util.Future +import com.twitter.util.Timer + +case class ProdConfig( + override val isServiceLocal: Boolean, + override val localConfigRepoPath: String, + override val inMemCacheOff: Boolean, + override val decider: Decider, + override val abDecider: LoggingABDecider, + override val featureSwitches: FeatureSwitches, + override val shardParams: ShardParams, + override val serviceIdentifier: ServiceIdentifier, + override val tunableMap: TunableMap, +)( + implicit val statsReceiver: StatsReceiver) + extends { + // Due to trait initialization logic in Scala, any abstract members declared in Config or + // DeployConfig should be declared in this block. Otherwise the abstract member might initialize to + // null if invoked before object creation finishing. + + val log = Logger("ProdConfig") + + // Deciders + val isPushserviceCanaryDeepbirdv2CanaryClusterEnabled = decider + .feature(DeciderKey.enablePushserviceDeepbirdv2CanaryClusterDeciderKey.toString).isAvailable + + // Client ids + val notifierThriftClientId = ClientId("frigate-notifier.prod") + val loggedOutNotifierThriftClientId = ClientId("frigate-logged-out-notifier.prod") + val pushserviceThriftClientId: ClientId = ClientId("frigate-pushservice.prod") + + // Dests + val frigateHistoryCacheDest = "/s/cache/frigate_history" + val memcacheCASDest = "/s/cache/magic_recs_cas:twemcaches" + val historyStoreMemcacheDest = + "/srv#/prod/local/cache/magic_recs_history:twemcaches" + + val deepbirdv2PredictionServiceDest = + if (serviceIdentifier.service.equals("frigate-pushservice-canary") && + isPushserviceCanaryDeepbirdv2CanaryClusterEnabled) + "/s/frigate/deepbirdv2-magicrecs-canary" + else "/s/frigate/deepbirdv2-magicrecs" + + override val fanoutMetadataColumn = "frigate/magicfanout/prod/mh/fanoutMetadata" + + override val timer: Timer = DefaultTimer + override val featureStoreUtil = FeatureStoreUtil.withParams(Some(serviceIdentifier)) + override val targetLevelFeaturesConfig = TargetLevelFeaturesConfig() + val pushServiceMHCacheDest = "/s/cache/pushservice_mh" + + val pushServiceCoreSvcsCacheDest = "/srv#/prod/local/cache/pushservice_core_svcs" + + val userTweetEntityGraphDest = "/s/cassowary/user_tweet_entity_graph" + val userUserGraphDest = "/s/cassowary/user_user_graph" + val lexServiceDest = "/s/live-video/timeline-thrift" + val entityGraphCacheDest = "/s/cache/pushservice_entity_graph" + + override val pushIbisV2Store = { + val service = Finagle.readOnlyThriftService( + "ibis-v2-service", + "/s/ibis2/ibis2", + statsReceiver, + notifierThriftClientId, + requestTimeout = 3.seconds, + tries = 3, + mTLSServiceIdentifier = Some(serviceIdentifier) + ) + + // according to ibis team, it is safe to retry on timeout, write & channel closed exceptions. + val pushIbisClient = new Ibis2Service.FinagledClient( + new DynamicRequestMeterFilter( + tunableMap(PushServiceTunableKeys.IbisQpsLimitTunableKey), + RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), + PushQPSLimitConstants.IbisOrNTabQPSForRFPH + )(timer).andThen(service), + RichClientParam(serviceName = "ibis-v2-service") + ) + + PushIbis2Store(pushIbisClient) + } + + val notificationServiceClient: NotificationService$FinagleClient = { + val service = Finagle.readWriteThriftService( + "notificationservice", + "/s/notificationservice/notificationservice", + statsReceiver, + pushserviceThriftClientId, + requestTimeout = 10.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ) + + new NotificationService.FinagledClient( + new DynamicRequestMeterFilter( + tunableMap(PushServiceTunableKeys.NtabQpsLimitTunableKey), + RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), + PushQPSLimitConstants.IbisOrNTabQPSForRFPH)(timer).andThen(service), + RichClientParam(serviceName = "notificationservice") + ) + } + + val notificationServiceApiClient: NotificationApi$FinagleClient = { + val service = Finagle.readWriteThriftService( + "notificationservice-api", + "/s/notificationservice/notificationservice-api:thrift", + statsReceiver, + pushserviceThriftClientId, + requestTimeout = 10.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ) + + new NotificationApi.FinagledClient( + new DynamicRequestMeterFilter( + tunableMap(PushServiceTunableKeys.NtabQpsLimitTunableKey), + RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), + PushQPSLimitConstants.IbisOrNTabQPSForRFPH)(timer).andThen(service), + RichClientParam(serviceName = "notificationservice-api") + ) + } + + val mrRequestScriberNode = "mr_request_scribe" + val loggedOutMrRequestScriberNode = "lo_mr_request_scribe" + + override val pushSendEventStreamName = "frigate_pushservice_send_event_prod" +} with DeployConfig { + // Scribe + private val notificationScribeLog = Logger("notification_scribe") + private val notificationScribeInjection: Injection[NotificationScribe, String] = BinaryScalaCodec( + NotificationScribe + ) andThen Injection.connect[Array[Byte], Base64String, String] + + override def notificationScribe(data: NotificationScribe): Unit = { + val logEntry: String = notificationScribeInjection(data) + notificationScribeLog.info(logEntry) + } + + // History Store - Invalidates cached history after writes + override val historyStore = new InvalidatingAfterWritesPushServiceHistoryStore( + ManhattanHistoryStore(notificationHistoryStore, statsReceiver), + recentHistoryCacheClient, + new DeciderGateBuilder(decider) + .idGate(DeciderKey.enableInvalidatingCachedHistoryStoreAfterWrites) + ) + + override val emailHistoryStore: PushServiceHistoryStore = { + statsReceiver.scope("frigate_email_history").counter("request").incr() + new SimplePushServiceHistoryStore(emailNotificationHistoryStore) + } + + override val loggedOutHistoryStore = + new InvalidatingAfterWritesPushServiceHistoryStore( + ManhattanKVHistoryStore( + manhattanKVLoggedOutHistoryStoreEndpoint, + "frigate_notification_logged_out_history"), + recentHistoryCacheClient, + new DeciderGateBuilder(decider) + .idGate(DeciderKey.enableInvalidatingCachedLoggedOutHistoryStoreAfterWrites) + ) + + private val requestScribeLog = Logger("request_scribe") + private val requestScribeInjection: Injection[PushRequestScribe, String] = BinaryScalaCodec( + PushRequestScribe + ) andThen Injection.connect[Array[Byte], Base64String, String] + + override def requestScribe(data: PushRequestScribe): Unit = { + val logEntry: String = requestScribeInjection(data) + requestScribeLog.info(logEntry) + } + + // generic notification server + override def notificationServiceSend( + target: Target, + request: CreateGenericNotificationRequest + ): Future[CreateGenericNotificationResponse] = + notificationServiceClient.createGenericNotification(request) + + // generic notification server + override def notificationServiceDelete( + request: DeleteGenericNotificationRequest + ): Future[Unit] = notificationServiceClient.deleteGenericNotification(request) + + // NTab-api + override def notificationServiceDeleteTimeline( + request: DeleteCurrentTimelineForUserRequest + ): Future[Unit] = notificationServiceApiClient.deleteCurrentTimelineForUser(request) + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.scala new file mode 100644 index 0000000000..c93ca0ea8b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/StagingConfig.scala @@ -0,0 +1,193 @@ +package com.twitter.frigate.pushservice.config + +import com.twitter.abdecider.LoggingABDecider +import com.twitter.conversions.DurationOps._ +import com.twitter.decider.Decider +import com.twitter.featureswitches.v2.FeatureSwitches +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.finagle.thrift.RichClientParam +import com.twitter.finagle.util.DefaultTimer +import com.twitter.frigate.common.config.RateLimiterGenerator +import com.twitter.frigate.common.filter.DynamicRequestMeterFilter +import com.twitter.frigate.common.history.InvalidatingAfterWritesPushServiceHistoryStore +import com.twitter.frigate.common.history.ManhattanHistoryStore +import com.twitter.frigate.common.history.ManhattanKVHistoryStore +import com.twitter.frigate.common.history.ReadOnlyHistoryStore +import com.twitter.frigate.common.history.PushServiceHistoryStore +import com.twitter.frigate.common.history.SimplePushServiceHistoryStore +import com.twitter.frigate.common.util.Finagle +import com.twitter.frigate.data_pipeline.features_common.FeatureStoreUtil +import com.twitter.frigate.data_pipeline.features_common.TargetLevelFeaturesConfig +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.DeciderKey +import com.twitter.frigate.pushservice.params.PushQPSLimitConstants +import com.twitter.frigate.pushservice.params.PushServiceTunableKeys +import com.twitter.frigate.pushservice.params.ShardParams +import com.twitter.frigate.pushservice.store._ +import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe +import com.twitter.frigate.scribe.thriftscala.NotificationScribe +import com.twitter.ibis2.service.thriftscala.Ibis2Service +import com.twitter.logging.Logger +import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponseType +import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest +import com.twitter.notificationservice.thriftscala.NotificationService +import com.twitter.notificationservice.thriftscala.NotificationService$FinagleClient +import com.twitter.servo.decider.DeciderGateBuilder +import com.twitter.util.tunable.TunableMap +import com.twitter.util.Future +import com.twitter.util.Timer + +case class StagingConfig( + override val isServiceLocal: Boolean, + override val localConfigRepoPath: String, + override val inMemCacheOff: Boolean, + override val decider: Decider, + override val abDecider: LoggingABDecider, + override val featureSwitches: FeatureSwitches, + override val shardParams: ShardParams, + override val serviceIdentifier: ServiceIdentifier, + override val tunableMap: TunableMap, +)( + implicit val statsReceiver: StatsReceiver) + extends { + // Due to trait initialization logic in Scala, any abstract members declared in Config or + // DeployConfig should be declared in this block. Otherwise the abstract member might initialize to + // null if invoked before object creation finishing. + + val log = Logger("StagingConfig") + + // Client ids + val notifierThriftClientId = ClientId("frigate-notifier.dev") + val loggedOutNotifierThriftClientId = ClientId("frigate-logged-out-notifier.dev") + val pushserviceThriftClientId: ClientId = ClientId("frigate-pushservice.staging") + + override val fanoutMetadataColumn = "frigate/magicfanout/staging/mh/fanoutMetadata" + + // dest + val frigateHistoryCacheDest = "/srv#/test/local/cache/twemcache_frigate_history" + val memcacheCASDest = "/srv#/test/local/cache/twemcache_magic_recs_cas_dev:twemcaches" + val pushServiceMHCacheDest = "/srv#/test/local/cache/twemcache_pushservice_test" + val entityGraphCacheDest = "/srv#/test/local/cache/twemcache_pushservice_test" + val pushServiceCoreSvcsCacheDest = "/srv#/test/local/cache/twemcache_pushservice_core_svcs_test" + val historyStoreMemcacheDest = "/srv#/test/local/cache/twemcache_eventstream_test:twemcaches" + val userTweetEntityGraphDest = "/cluster/local/cassowary/staging/user_tweet_entity_graph" + val userUserGraphDest = "/cluster/local/cassowary/staging/user_user_graph" + val lexServiceDest = "/srv#/staging/local/live-video/timeline-thrift" + val deepbirdv2PredictionServiceDest = "/cluster/local/frigate/staging/deepbirdv2-magicrecs" + + override val featureStoreUtil = FeatureStoreUtil.withParams(Some(serviceIdentifier)) + override val targetLevelFeaturesConfig = TargetLevelFeaturesConfig() + val mrRequestScriberNode = "validation_mr_request_scribe" + val loggedOutMrRequestScriberNode = "lo_mr_request_scribe" + + override val timer: Timer = DefaultTimer + + override val pushSendEventStreamName = "frigate_pushservice_send_event_staging" + + override val pushIbisV2Store = { + val service = Finagle.readWriteThriftService( + "ibis-v2-service", + "/s/ibis2/ibis2", + statsReceiver, + notifierThriftClientId, + requestTimeout = 6.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ) + + val pushIbisClient = new Ibis2Service.FinagledClient( + new DynamicRequestMeterFilter( + tunableMap(PushServiceTunableKeys.IbisQpsLimitTunableKey), + RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), + PushQPSLimitConstants.IbisOrNTabQPSForRFPH + )(timer).andThen(service), + RichClientParam(serviceName = "ibis-v2-service") + ) + + StagingIbis2Store(PushIbis2Store(pushIbisClient)) + } + + val notificationServiceClient: NotificationService$FinagleClient = { + val service = Finagle.readWriteThriftService( + "notificationservice", + "/s/notificationservice/notificationservice", + statsReceiver, + pushserviceThriftClientId, + requestTimeout = 10.seconds, + mTLSServiceIdentifier = Some(serviceIdentifier) + ) + + new NotificationService.FinagledClient( + new DynamicRequestMeterFilter( + tunableMap(PushServiceTunableKeys.NtabQpsLimitTunableKey), + RateLimiterGenerator.asTuple(_, shardParams.numShards, 20), + PushQPSLimitConstants.IbisOrNTabQPSForRFPH)(timer).andThen(service), + RichClientParam(serviceName = "notificationservice") + ) + } +} with DeployConfig { + + // Scribe + private val notificationScribeLog = Logger("StagingNotificationScribe") + + override def notificationScribe(data: NotificationScribe): Unit = { + notificationScribeLog.info(data.toString) + } + private val requestScribeLog = Logger("StagingRequestScribe") + + override def requestScribe(data: PushRequestScribe): Unit = { + requestScribeLog.info(data.toString) + } + + // history store + override val historyStore = new InvalidatingAfterWritesPushServiceHistoryStore( + ReadOnlyHistoryStore( + ManhattanHistoryStore(notificationHistoryStore, statsReceiver) + ), + recentHistoryCacheClient, + new DeciderGateBuilder(decider) + .idGate(DeciderKey.enableInvalidatingCachedHistoryStoreAfterWrites) + ) + + override val emailHistoryStore: PushServiceHistoryStore = new SimplePushServiceHistoryStore( + emailNotificationHistoryStore) + + // history store + override val loggedOutHistoryStore = + new InvalidatingAfterWritesPushServiceHistoryStore( + ReadOnlyHistoryStore( + ManhattanKVHistoryStore( + manhattanKVLoggedOutHistoryStoreEndpoint, + "frigate_notification_logged_out_history")), + recentHistoryCacheClient, + new DeciderGateBuilder(decider) + .idGate(DeciderKey.enableInvalidatingCachedLoggedOutHistoryStoreAfterWrites) + ) + + override def notificationServiceSend( + target: Target, + request: CreateGenericNotificationRequest + ): Future[CreateGenericNotificationResponse] = + target.isTeamMember.flatMap { isTeamMember => + if (isTeamMember) { + notificationServiceClient.createGenericNotification(request) + } else { + log.info(s"Mock creating generic notification $request for user: ${target.targetId}") + Future.value( + CreateGenericNotificationResponse(CreateGenericNotificationResponseType.Success) + ) + } + } + + override def notificationServiceDelete( + request: DeleteGenericNotificationRequest + ): Future[Unit] = Future.Unit + + override def notificationServiceDeleteTimeline( + request: DeleteCurrentTimelineForUserRequest + ): Future[Unit] = Future.Unit +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.scala new file mode 100644 index 0000000000..c45ccc72ef --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/config/mlconfig/DeepbirdV2ModelConfig.scala @@ -0,0 +1,23 @@ +package com.twitter.frigate.pushservice.config.mlconfig + +import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.ml.prediction.DeepbirdPredictionEngineServiceStore +import com.twitter.nrel.heavyranker.PushDBv2PredictionServiceStore + +object DeepbirdV2ModelConfig { + def buildPredictionServiceScoreStore( + predictionServiceClient: DeepbirdPredictionService.ServiceToClient, + serviceName: String + )( + implicit statsReceiver: StatsReceiver + ): PushDBv2PredictionServiceStore = { + + val stats = statsReceiver.scope(serviceName) + val serviceStats = statsReceiver.scope("dbv2PredictionServiceStore") + + new PushDBv2PredictionServiceStore( + DeepbirdPredictionEngineServiceStore(predictionServiceClient, batchSize = Some(32))(stats) + )(serviceStats) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.scala new file mode 100644 index 0000000000..d271e5a576 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/controller/PushServiceController.scala @@ -0,0 +1,114 @@ +package com.twitter.frigate.pushservice.controller + +import com.google.inject.Inject +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.thrift.ClientId +import com.twitter.finatra.thrift.Controller +import com.twitter.frigate.pushservice.exception.DisplayLocationNotSupportedException +import com.twitter.frigate.pushservice.refresh_handler.RefreshForPushHandler +import com.twitter.frigate.pushservice.send_handler.SendHandler +import com.twitter.frigate.pushservice.refresh_handler.LoggedOutRefreshForPushHandler +import com.twitter.frigate.pushservice.thriftscala.PushService.Loggedout +import com.twitter.frigate.pushservice.thriftscala.PushService.Refresh +import com.twitter.frigate.pushservice.thriftscala.PushService.Send +import com.twitter.frigate.pushservice.{thriftscala => t} +import com.twitter.frigate.thriftscala.NotificationDisplayLocation +import com.twitter.util.logging.Logging +import com.twitter.util.Future + +class PushServiceController @Inject() ( + sendHandler: SendHandler, + refreshForPushHandler: RefreshForPushHandler, + loggedOutRefreshForPushHandler: LoggedOutRefreshForPushHandler, + statsReceiver: StatsReceiver) + extends Controller(t.PushService) + with Logging { + + private val stats: StatsReceiver = statsReceiver.scope(s"${this.getClass.getSimpleName}") + private val failureCount = stats.counter("failures") + private val failureStatsScope = stats.scope("failures") + private val uncaughtErrorCount = failureStatsScope.counter("uncaught") + private val uncaughtErrorScope = failureStatsScope.scope("uncaught") + private val clientIdScope = stats.scope("client_id") + + handle(t.PushService.Send) { request: Send.Args => + send(request) + } + + handle(t.PushService.Refresh) { args: Refresh.Args => + refresh(args) + } + + handle(t.PushService.Loggedout) { request: Loggedout.Args => + loggedOutRefresh(request) + } + + private def loggedOutRefresh( + request: t.PushService.Loggedout.Args + ): Future[t.PushService.Loggedout.SuccessType] = { + val fut = request.request.notificationDisplayLocation match { + case NotificationDisplayLocation.PushToMobileDevice => + loggedOutRefreshForPushHandler.refreshAndSend(request.request) + case _ => + Future.exception( + new DisplayLocationNotSupportedException( + "Specified notification display location is not supported")) + } + fut.onFailure { ex => + logger.error( + s"Failure in push service for logged out refresh request: $request - ${ex.getMessage} - ${ex.getStackTrace + .mkString(", \n\t")}", + ex) + failureCount.incr() + uncaughtErrorCount.incr() + uncaughtErrorScope.counter(ex.getClass.getCanonicalName).incr() + } + } + + private def refresh( + request: t.PushService.Refresh.Args + ): Future[t.PushService.Refresh.SuccessType] = { + + val fut = request.request.notificationDisplayLocation match { + case NotificationDisplayLocation.PushToMobileDevice => + val clientId: String = + ClientId.current + .flatMap { cid => Option(cid.name) } + .getOrElse("none") + clientIdScope.counter(clientId).incr() + refreshForPushHandler.refreshAndSend(request.request) + case _ => + Future.exception( + new DisplayLocationNotSupportedException( + "Specified notification display location is not supported")) + } + fut.onFailure { ex => + logger.error( + s"Failure in push service for refresh request: $request - ${ex.getMessage} - ${ex.getStackTrace + .mkString(", \n\t")}", + ex + ) + + failureCount.incr() + uncaughtErrorCount.incr() + uncaughtErrorScope.counter(ex.getClass.getCanonicalName).incr() + } + + } + + private def send( + request: t.PushService.Send.Args + ): Future[t.PushService.Send.SuccessType] = { + sendHandler(request.request).onFailure { ex => + logger.error( + s"Failure in push service for send request: $request - ${ex.getMessage} - ${ex.getStackTrace + .mkString(", \n\t")}", + ex + ) + + failureCount.incr() + uncaughtErrorCount.incr() + uncaughtErrorScope.counter(ex.getClass.getCanonicalName).incr() + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.scala new file mode 100644 index 0000000000..08399c934e --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/DisplayLocationNotSupportedException.scala @@ -0,0 +1,12 @@ +package com.twitter.frigate.pushservice.exception + +import scala.util.control.NoStackTrace + +/** + * Throw exception if DisplayLocation is not supported + * + * @param message Exception message + */ +class DisplayLocationNotSupportedException(private val message: String) + extends Exception(message) + with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.scala new file mode 100644 index 0000000000..8f0d2b9882 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/InvalidSportDomainException.scala @@ -0,0 +1,12 @@ +package com.twitter.frigate.pushservice.exception + +import scala.util.control.NoStackTrace + +/** + * Throw exception if the sport domain is not supported by MagicFanoutSports + * + * @param message Exception message + */ +class InvalidSportDomainException(private val message: String) + extends Exception(message) + with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.scala new file mode 100644 index 0000000000..069e65d797 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/TweetNTabRequestHydratorException.scala @@ -0,0 +1,7 @@ +package com.twitter.frigate.pushservice.exception + +import scala.util.control.NoStackTrace + +class TweetNTabRequestHydratorException(private val message: String) + extends Exception(message) + with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.scala new file mode 100644 index 0000000000..5ed6c1c284 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UnsupportedCrtException.scala @@ -0,0 +1,11 @@ +package com.twitter.frigate.pushservice.exception + +import scala.util.control.NoStackTrace + +/** + * Exception for CRT not expected in the scope + * @param message Exception message to log the UnsupportedCrt + */ +class UnsupportedCrtException(private val message: String) + extends Exception(message) + with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.scala new file mode 100644 index 0000000000..3ac069dac7 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/exception/UttEntityNotFoundException.scala @@ -0,0 +1,12 @@ +package com.twitter.frigate.pushservice.exception + +import scala.util.control.NoStackTrace + +/** + * Throw exception if UttEntity is not found where it might be a required data field + * + * @param message Exception message + */ +class UttEntityNotFoundException(private val message: String) + extends Exception(message) + with NoStackTrace diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.scala new file mode 100644 index 0000000000..addf5b4383 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HealthFeatureGetter.scala @@ -0,0 +1,220 @@ +package com.twitter.frigate.pushservice.ml + +import com.twitter.abuse.detection.scoring.thriftscala.{Model => TweetHealthModel} +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse +import com.twitter.frigate.common.base.FeatureMap +import com.twitter.frigate.common.base.TweetAuthor +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.predicate.HealthPredicates.userHealthSignalValueToDouble +import com.twitter.frigate.pushservice.util.CandidateHydrationUtil +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil +import com.twitter.frigate.thriftscala.UserMediaRepresentation +import com.twitter.hss.api.thriftscala.SignalValue +import com.twitter.hss.api.thriftscala.UserHealthSignal +import com.twitter.hss.api.thriftscala.UserHealthSignal.AgathaCalibratedNsfwDouble +import com.twitter.hss.api.thriftscala.UserHealthSignal.NsfwTextUserScoreDouble +import com.twitter.hss.api.thriftscala.UserHealthSignalResponse +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future +import com.twitter.util.Time + +object HealthFeatureGetter { + + def getFeatures( + pushCandidate: PushCandidate, + producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation], + userHealthScoreStore: ReadableStore[Long, UserHealthSignalResponse], + tweetHealthScoreStoreOpt: Option[ReadableStore[TweetScoringRequest, TweetScoringResponse]] = + None + ): Future[FeatureMap] = { + + pushCandidate match { + case cand: PushCandidate with TweetCandidate with TweetAuthor with TweetAuthorDetails => + val pMediaNsfwRequest = + TweetScoringRequest(cand.tweetId, TweetHealthModel.ExperimentalHealthModelScore4) + val pTweetTextNsfwRequest = + TweetScoringRequest(cand.tweetId, TweetHealthModel.ExperimentalHealthModelScore1) + + cand.authorId match { + case Some(authorId) => + Future + .join( + userHealthScoreStore.get(authorId), + producerMediaRepresentationStore.get(authorId), + tweetHealthScoreStoreOpt.map(_.get(pMediaNsfwRequest)).getOrElse(Future.None), + tweetHealthScoreStoreOpt.map(_.get(pTweetTextNsfwRequest)).getOrElse(Future.None), + cand.tweetAuthor + ).map { + case ( + healthSignalsResponseOpt, + producerMuOpt, + pMediaNsfwOpt, + pTweetTextNsfwOpt, + tweetAuthorOpt) => + val healthSignalScoreMap = healthSignalsResponseOpt + .map(_.signalValues).getOrElse(Map.empty[UserHealthSignal, SignalValue]) + val agathaNSFWScore = userHealthSignalValueToDouble( + healthSignalScoreMap + .getOrElse(AgathaCalibratedNsfwDouble, SignalValue.DoubleValue(0.5))) + val userTextNSFWScore = userHealthSignalValueToDouble( + healthSignalScoreMap + .getOrElse(NsfwTextUserScoreDouble, SignalValue.DoubleValue(0.15))) + val pMediaNsfwScore = pMediaNsfwOpt.map(_.score).getOrElse(0.0) + val pTweetTextNsfwScore = pTweetTextNsfwOpt.map(_.score).getOrElse(0.0) + + val mediaRepresentationMap = + producerMuOpt.map(_.mediaRepresentation).getOrElse(Map.empty[String, Double]) + val sumScore: Double = mediaRepresentationMap.values.sum + val nudityRate = + if (sumScore > 0) + mediaRepresentationMap.getOrElse( + MediaAnnotationsUtil.nudityCategoryId, + 0.0) / sumScore + else 0.0 + val beautyRate = + if (sumScore > 0) + mediaRepresentationMap.getOrElse( + MediaAnnotationsUtil.beautyCategoryId, + 0.0) / sumScore + else 0.0 + val singlePersonRate = + if (sumScore > 0) + mediaRepresentationMap.getOrElse( + MediaAnnotationsUtil.singlePersonCategoryId, + 0.0) / sumScore + else 0.0 + val dislikeCt = cand.numericFeatures.getOrElse( + "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_ntab_disliked.any_feature.Duration.Top.count", + 0.0) + val sentCt = cand.numericFeatures.getOrElse( + "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_sent.any_feature.Duration.Top.count", + 0.0) + val dislikeRate = if (sentCt > 0) dislikeCt / sentCt else 0.0 + + val authorDislikeCt = cand.numericFeatures.getOrElse( + "tweet_author_aggregate.pair.label.ntab.isDisliked.any_feature.28.days.count", + 0.0) + val authorReportCt = cand.numericFeatures.getOrElse( + "tweet_author_aggregate.pair.label.reportTweetDone.any_feature.28.days.count", + 0.0) + val authorSentCt = cand.numericFeatures + .getOrElse( + "tweet_author_aggregate.pair.any_label.any_feature.28.days.count", + 0.0) + val authorDislikeRate = + if (authorSentCt > 0) authorDislikeCt / authorSentCt else 0.0 + val authorReportRate = + if (authorSentCt > 0) authorReportCt / authorSentCt else 0.0 + + val (isNsfwAccount, authorAccountAge) = tweetAuthorOpt match { + case Some(tweetAuthor) => + ( + CandidateHydrationUtil.isNsfwAccount( + tweetAuthor, + cand.target.params(PushFeatureSwitchParams.NsfwTokensParam)), + (Time.now - Time.fromMilliseconds(tweetAuthor.createdAtMsec)).inHours + ) + case _ => (false, 0) + } + + val tweetSemanticCoreIds = cand.sparseBinaryFeatures + .getOrElse(PushConstants.TweetSemanticCoreIdFeature, Set.empty[String]) + + val continuousFeatures = Map[String, Double]( + "agathaNsfwScore" -> agathaNSFWScore, + "textNsfwScore" -> userTextNSFWScore, + "pMediaNsfwScore" -> pMediaNsfwScore, + "pTweetTextNsfwScore" -> pTweetTextNsfwScore, + "nudityRate" -> nudityRate, + "beautyRate" -> beautyRate, + "singlePersonRate" -> singlePersonRate, + "numSources" -> CandidateUtil.getTagsCRCount(cand), + "favCount" -> cand.numericFeatures + .getOrElse("tweet.core.tweet_counts.favorite_count", 0.0), + "activeFollowers" -> cand.numericFeatures + .getOrElse("RecTweetAuthor.User.ActiveFollowers", 0.0), + "favorsRcvd28Days" -> cand.numericFeatures + .getOrElse("RecTweetAuthor.User.FavorsRcvd28Days", 0.0), + "tweets28Days" -> cand.numericFeatures + .getOrElse("RecTweetAuthor.User.Tweets28Days", 0.0), + "dislikeCount" -> dislikeCt, + "dislikeRate" -> dislikeRate, + "sentCount" -> sentCt, + "authorDislikeCount" -> authorDislikeCt, + "authorDislikeRate" -> authorDislikeRate, + "authorReportCount" -> authorReportCt, + "authorReportRate" -> authorReportRate, + "authorSentCount" -> authorSentCt, + "authorAgeInHour" -> authorAccountAge.toDouble + ) + + val booleanFeatures = Map[String, Boolean]( + "isSimclusterBased" -> RecTypes.simclusterBasedTweets + .contains(cand.commonRecType), + "isTopicTweet" -> RecTypes.isTopicTweetType(cand.commonRecType), + "isHashSpace" -> RecTypes.tagspaceTypes.contains(cand.commonRecType), + "isFRS" -> RecTypes.frsTypes.contains(cand.commonRecType), + "isModelingBased" -> RecTypes.mrModelingBasedTypes.contains(cand.commonRecType), + "isGeoPop" -> RecTypes.GeoPopTweetTypes.contains(cand.commonRecType), + "hasPhoto" -> cand.booleanFeatures + .getOrElse("RecTweet.TweetyPieResult.HasPhoto", false), + "hasVideo" -> cand.booleanFeatures + .getOrElse("RecTweet.TweetyPieResult.HasVideo", false), + "hasUrl" -> cand.booleanFeatures + .getOrElse("RecTweet.TweetyPieResult.HasUrl", false), + "isMrTwistly" -> CandidateUtil.isMrTwistlyCandidate(cand), + "abuseStrikeTop2Percent" -> tweetSemanticCoreIds.contains( + PushConstants.AbuseStrike_Top2Percent_Id), + "abuseStrikeTop1Percent" -> tweetSemanticCoreIds.contains( + PushConstants.AbuseStrike_Top1Percent_Id), + "abuseStrikeTop05Percent" -> tweetSemanticCoreIds.contains( + PushConstants.AbuseStrike_Top05Percent_Id), + "abuseStrikeTop025Percent" -> tweetSemanticCoreIds.contains( + PushConstants.AbuseStrike_Top025Percent_Id), + "allSpamReportsPerFavTop1Percent" -> tweetSemanticCoreIds.contains( + PushConstants.AllSpamReportsPerFav_Top1Percent_Id), + "reportsPerFavTop1Percent" -> tweetSemanticCoreIds.contains( + PushConstants.ReportsPerFav_Top1Percent_Id), + "reportsPerFavTop2Percent" -> tweetSemanticCoreIds.contains( + PushConstants.ReportsPerFav_Top2Percent_Id), + "isNudity" -> tweetSemanticCoreIds.contains( + PushConstants.MediaUnderstanding_Nudity_Id), + "beautyStyleFashion" -> tweetSemanticCoreIds.contains( + PushConstants.MediaUnderstanding_Beauty_Id), + "singlePerson" -> tweetSemanticCoreIds.contains( + PushConstants.MediaUnderstanding_SinglePerson_Id), + "pornList" -> tweetSemanticCoreIds.contains(PushConstants.PornList_Id), + "pornographyAndNsfwContent" -> tweetSemanticCoreIds.contains( + PushConstants.PornographyAndNsfwContent_Id), + "sexLife" -> tweetSemanticCoreIds.contains(PushConstants.SexLife_Id), + "sexLifeOrSexualOrientation" -> tweetSemanticCoreIds.contains( + PushConstants.SexLifeOrSexualOrientation_Id), + "profanity" -> tweetSemanticCoreIds.contains(PushConstants.ProfanityFilter_Id), + "isVerified" -> cand.booleanFeatures + .getOrElse("RecTweetAuthor.User.IsVerified", false), + "hasNsfwToken" -> isNsfwAccount + ) + + val stringFeatures = Map[String, String]( + "tweetLanguage" -> cand.categoricalFeatures + .getOrElse("tweet.core.tweet_text.language", "") + ) + + FeatureMap( + booleanFeatures = booleanFeatures, + numericFeatures = continuousFeatures, + categoricalFeatures = stringFeatures) + } + case _ => Future.value(FeatureMap()) + } + case _ => Future.value(FeatureMap()) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.scala new file mode 100644 index 0000000000..023adb81e3 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/HydrationContextBuilder.scala @@ -0,0 +1,179 @@ +package com.twitter.frigate.pushservice.ml + +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.ml.feature.TweetSocialProofKey +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.predicate.quality_model_predicate.PDauCohortUtil +import com.twitter.nrel.hydration.base.FeatureInput +import com.twitter.nrel.hydration.push.HydrationContext +import com.twitter.nrel.hydration.frigate.{FeatureInputs => FI} +import com.twitter.util.Future + +object HydrationContextBuilder { + + private def getRecUserInputs( + pushCandidate: PushCandidate + ): Set[FI.RecUser] = { + pushCandidate match { + case userCandidate: UserCandidate => + Set(FI.RecUser(userCandidate.userId)) + case _ => Set.empty + } + } + + private def getRecTweetInputs( + pushCandidate: PushCandidate + ): Set[FI.RecTweet] = + pushCandidate match { + case tweetCandidateWithAuthor: TweetCandidate with TweetAuthor with TweetAuthorDetails => + val authorIdOpt = tweetCandidateWithAuthor.authorId + Set(FI.RecTweet(tweetCandidateWithAuthor.tweetId, authorIdOpt)) + case _ => Set.empty + } + + private def getMediaInputs( + pushCandidate: PushCandidate + ): Set[FI.Media] = + pushCandidate match { + case tweetCandidateWithMedia: TweetCandidate with TweetDetails => + tweetCandidateWithMedia.mediaKeys + .map { mk => + Set(FI.Media(mk)) + }.getOrElse(Set.empty) + case _ => Set.empty + } + + private def getEventInputs( + pushCandidate: PushCandidate + ): Set[FI.Event] = pushCandidate match { + case mrEventCandidate: EventCandidate => + Set(FI.Event(mrEventCandidate.eventId)) + case mfEventCandidate: MagicFanoutEventCandidate => + Set(FI.Event(mfEventCandidate.eventId)) + case _ => Set.empty + } + + private def getTopicInputs( + pushCandidate: PushCandidate + ): Set[FI.Topic] = + pushCandidate match { + case mrTopicCandidate: TopicCandidate => + mrTopicCandidate.semanticCoreEntityId match { + case Some(topicId) => Set(FI.Topic(topicId)) + case _ => Set.empty + } + case _ => Set.empty + } + + private def getTweetSocialProofKey( + pushCandidate: PushCandidate + ): Future[Set[FI.SocialProofKey]] = { + pushCandidate match { + case candidate: TweetCandidate with SocialContextActions => + val target = pushCandidate.target + target.seedsWithWeight.map { seedsWithWeightOpt => + Set( + FI.SocialProofKey( + TweetSocialProofKey( + seedsWithWeightOpt.getOrElse(Map.empty), + candidate.socialContextAllTypeActions + )) + ) + } + case _ => Future.value(Set.empty) + } + } + + private def getSocialContextInputs( + pushCandidate: PushCandidate + ): Future[Set[FeatureInput]] = + pushCandidate match { + case candidateWithSC: Candidate with SocialContextActions => + val tweetSocialProofKeyFut = getTweetSocialProofKey(pushCandidate) + tweetSocialProofKeyFut.map { tweetSocialProofKeyOpt => + val socialContextUsers = FI.SocialContextUsers(candidateWithSC.socialContextUserIds.toSet) + val socialContextActions = + FI.SocialContextActions(candidateWithSC.socialContextAllTypeActions) + val socialProofKeyOpt = tweetSocialProofKeyOpt + Set(Set(socialContextUsers), Set(socialContextActions), socialProofKeyOpt).flatten + } + case _ => Future.value(Set.empty) + } + + private def getPushStringGroupInputs( + pushCandidate: PushCandidate + ): Set[FI.PushStringGroup] = + Set( + FI.PushStringGroup( + pushCandidate.getPushCopy.flatMap(_.pushStringGroup).map(_.toString).getOrElse("") + )) + + private def getCRTInputs( + pushCandidate: PushCandidate + ): Set[FI.CommonRecommendationType] = + Set(FI.CommonRecommendationType(pushCandidate.commonRecType)) + + private def getFrigateNotification( + pushCandidate: PushCandidate + ): Set[FI.CandidateFrigateNotification] = + Set(FI.CandidateFrigateNotification(pushCandidate.frigateNotification)) + + private def getCopyId( + pushCandidate: PushCandidate + ): Set[FI.CopyId] = + Set(FI.CopyId(pushCandidate.pushCopyId, pushCandidate.ntabCopyId)) + + def build(candidate: PushCandidate): Future[HydrationContext] = { + val socialContextInputsFut = getSocialContextInputs(candidate) + socialContextInputsFut.map { socialContextInputs => + val featureInputs: Set[FeatureInput] = + socialContextInputs ++ + getRecUserInputs(candidate) ++ + getRecTweetInputs(candidate) ++ + getEventInputs(candidate) ++ + getTopicInputs(candidate) ++ + getCRTInputs(candidate) ++ + getPushStringGroupInputs(candidate) ++ + getMediaInputs(candidate) ++ + getFrigateNotification(candidate) ++ + getCopyId(candidate) + + HydrationContext( + candidate.target.targetId, + featureInputs + ) + } + } + + def build(target: Target): Future[HydrationContext] = { + val realGraphFeaturesFut = target.realGraphFeatures + for { + realGraphFeaturesOpt <- realGraphFeaturesFut + dauProb <- PDauCohortUtil.getDauProb(target) + mrUserStateOpt <- target.targetMrUserState + historyInputOpt <- + if (target.params(PushFeatureSwitchParams.EnableHydratingOnlineMRHistoryFeatures)) { + target.onlineLabeledPushRecs.map { mrHistoryValueOpt => + mrHistoryValueOpt.map(FI.MrHistory) + } + } else Future.None + } yield { + val realGraphFeaturesInputOpt = realGraphFeaturesOpt.map { realGraphFeatures => + FI.TargetRealGraphFeatures(realGraphFeatures) + } + val dauProbInput = FI.DauProb(dauProb) + val mrUserStateInput = FI.MrUserState(mrUserStateOpt.map(_.name).getOrElse("unknown")) + HydrationContext( + target.targetId, + Seq( + realGraphFeaturesInputOpt, + historyInputOpt, + Some(dauProbInput), + Some(mrUserStateInput) + ).flatten.toSet + ) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.scala new file mode 100644 index 0000000000..fd702cc3c7 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/ml/PushMLModelScorer.scala @@ -0,0 +1,188 @@ +package com.twitter.frigate.pushservice.ml + +import com.twitter.cortex.deepbird.thriftjava.ModelSelector +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.FeatureMap +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushMLModel +import com.twitter.frigate.pushservice.params.PushModelName +import com.twitter.frigate.pushservice.params.WeightedOpenOrNtabClickModel +import com.twitter.nrel.heavyranker.PushCandidateHydrationContextWithModel +import com.twitter.nrel.heavyranker.PushPredictionServiceStore +import com.twitter.nrel.heavyranker.TargetFeatureMapWithModel +import com.twitter.timelines.configapi.FSParam +import com.twitter.util.Future + +/** + * PushMLModelScorer scores the Candidates and populates their ML scores + * + * @param pushMLModel Enum to specify which model to use for scoring the Candidates + * @param modelToPredictionServiceStoreMap Supports all other prediction services. Specifies model ID -> dbv2 ReadableStore + * @param defaultDBv2PredictionServiceStore: Supports models that are not specified in the previous maps (which will be directly configured in the config repo) + * @param scoringStats StatsReceiver for scoping stats + */ +class PushMLModelScorer( + pushMLModel: PushMLModel.Value, + modelToPredictionServiceStoreMap: Map[ + WeightedOpenOrNtabClickModel.ModelNameType, + PushPredictionServiceStore + ], + defaultDBv2PredictionServiceStore: PushPredictionServiceStore, + scoringStats: StatsReceiver) { + + val queriesOutsideTheModelMaps: StatsReceiver = + scoringStats.scope("queries_outside_the_model_maps") + val totalQueriesOutsideTheModelMaps: Counter = + queriesOutsideTheModelMaps.counter("total") + + private def scoreByBatchPredictionForModelFromMultiModelService( + predictionServiceStore: PushPredictionServiceStore, + modelVersion: WeightedOpenOrNtabClickModel.ModelNameType, + candidatesDetails: Seq[CandidateDetails[PushCandidate]], + useCommonFeatures: Boolean, + overridePushMLModel: PushMLModel.Value + ): Seq[CandidateDetails[PushCandidate]] = { + val modelName = + PushModelName(overridePushMLModel, modelVersion).toString + val modelSelector = new ModelSelector() + modelSelector.setId(modelName) + + val candidateHydrationWithFeaturesMap = candidatesDetails.map { candidatesDetail => + ( + candidatesDetail.candidate.candidateHydrationContext, + candidatesDetail.candidate.candidateFeatureMap()) + } + if (candidatesDetails.nonEmpty) { + val candidatesWithScore = predictionServiceStore.getBatchPredictionsForModel( + candidatesDetails.head.candidate.target.targetHydrationContext, + candidatesDetails.head.candidate.target.featureMap, + candidateHydrationWithFeaturesMap, + Some(modelSelector), + useCommonFeatures + ) + candidatesDetails.zip(candidatesWithScore).foreach { + case (candidateDetail, (_, scoreOptFut)) => + candidateDetail.candidate.populateQualityModelScore( + overridePushMLModel, + modelVersion, + scoreOptFut + ) + } + } + + candidatesDetails + } + + private def scoreByBatchPrediction( + modelVersion: WeightedOpenOrNtabClickModel.ModelNameType, + candidatesDetails: Seq[CandidateDetails[PushCandidate]], + useCommonFeaturesForDBv2Service: Boolean, + overridePushMLModel: PushMLModel.Value + ): Seq[CandidateDetails[PushCandidate]] = { + if (modelToPredictionServiceStoreMap.contains(modelVersion)) { + scoreByBatchPredictionForModelFromMultiModelService( + modelToPredictionServiceStoreMap(modelVersion), + modelVersion, + candidatesDetails, + useCommonFeaturesForDBv2Service, + overridePushMLModel + ) + } else { + totalQueriesOutsideTheModelMaps.incr() + queriesOutsideTheModelMaps.counter(modelVersion).incr() + scoreByBatchPredictionForModelFromMultiModelService( + defaultDBv2PredictionServiceStore, + modelVersion, + candidatesDetails, + useCommonFeaturesForDBv2Service, + overridePushMLModel + ) + } + } + + def scoreByBatchPredictionForModelVersion( + target: Target, + candidatesDetails: Seq[CandidateDetails[PushCandidate]], + modelVersionParam: FSParam[WeightedOpenOrNtabClickModel.ModelNameType], + useCommonFeaturesForDBv2Service: Boolean = true, + overridePushMLModelOpt: Option[PushMLModel.Value] = None + ): Seq[CandidateDetails[PushCandidate]] = { + scoreByBatchPrediction( + target.params(modelVersionParam), + candidatesDetails, + useCommonFeaturesForDBv2Service, + overridePushMLModelOpt.getOrElse(pushMLModel) + ) + } + + def singlePredicationForModelVersion( + modelVersion: String, + candidate: PushCandidate, + overridePushMLModelOpt: Option[PushMLModel.Value] = None + ): Future[Option[Double]] = { + val modelSelector = new ModelSelector() + modelSelector.setId( + PushModelName(overridePushMLModelOpt.getOrElse(pushMLModel), modelVersion).toString + ) + if (modelToPredictionServiceStoreMap.contains(modelVersion)) { + modelToPredictionServiceStoreMap(modelVersion).get( + PushCandidateHydrationContextWithModel( + candidate.target.targetHydrationContext, + candidate.target.featureMap, + candidate.candidateHydrationContext, + candidate.candidateFeatureMap(), + Some(modelSelector) + ) + ) + } else { + totalQueriesOutsideTheModelMaps.incr() + queriesOutsideTheModelMaps.counter(modelVersion).incr() + defaultDBv2PredictionServiceStore.get( + PushCandidateHydrationContextWithModel( + candidate.target.targetHydrationContext, + candidate.target.featureMap, + candidate.candidateHydrationContext, + candidate.candidateFeatureMap(), + Some(modelSelector) + ) + ) + } + } + + def singlePredictionForTargetLevel( + modelVersion: String, + targetId: Long, + featureMap: Future[FeatureMap] + ): Future[Option[Double]] = { + val modelSelector = new ModelSelector() + modelSelector.setId( + PushModelName(pushMLModel, modelVersion).toString + ) + defaultDBv2PredictionServiceStore.getForTargetLevel( + TargetFeatureMapWithModel(targetId, featureMap, Some(modelSelector)) + ) + } + + def getScoreHistogramCounters( + stats: StatsReceiver, + scopeName: String, + histogramBinSize: Double + ): IndexedSeq[Counter] = { + val histogramScopedStatsReceiver = stats.scope(scopeName) + val numBins = math.ceil(1.0 / histogramBinSize).toInt + + (0 to numBins) map { k => + if (k == 0) + histogramScopedStatsReceiver.counter("candidates_with_scores_zero") + else { + val counterName = "candidates_with_scores_from_%s_to_%s".format( + "%.2f".format(histogramBinSize * (k - 1)).replace(".", ""), + "%.2f".format(math.min(1.0, histogramBinSize * k)).replace(".", "")) + histogramScopedStatsReceiver.counter(counterName) + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.scala new file mode 100644 index 0000000000..dc350a7404 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/DiscoverTwitter.scala @@ -0,0 +1,89 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.DiscoverTwitterCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.DiscoverTwitterPushIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.DiscoverTwitterNtabRequestHydrator +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.frigate.pushservice.take.predicates.BasicRFPHPredicates +import com.twitter.frigate.pushservice.take.predicates.OutOfNetworkTweetPredicates +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.hermit.predicate.NamedPredicate + +class DiscoverTwitterPushCandidate( + candidate: RawCandidate with DiscoverTwitterCandidate, + copyIds: CopyIds, +)( + implicit val statsScoped: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with DiscoverTwitterCandidate + with DiscoverTwitterPushIbis2Hydrator + with DiscoverTwitterNtabRequestHydrator { + + override val pushCopyId: Option[Int] = copyIds.pushCopyId + + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + + override val copyAggregationId: Option[String] = copyIds.aggregationId + + override val target: Target = candidate.target + + override lazy val commonRecType: CommonRecommendationType = candidate.commonRecType + + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + + override val statsReceiver: StatsReceiver = + statsScoped.scope("DiscoverTwitterPushCandidate") +} + +case class AddressBookPushCandidatePredicates(config: Config) + extends BasicRFPHPredicates[DiscoverTwitterPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val predicates: List[ + NamedPredicate[DiscoverTwitterPushCandidate] + ] = + List( + PredicatesForCandidate.paramPredicate( + PushFeatureSwitchParams.EnableAddressBookPush + ) + ) +} + +case class CompleteOnboardingPushCandidatePredicates(config: Config) + extends BasicRFPHPredicates[DiscoverTwitterPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val predicates: List[ + NamedPredicate[DiscoverTwitterPushCandidate] + ] = + List( + PredicatesForCandidate.paramPredicate( + PushFeatureSwitchParams.EnableCompleteOnboardingPush + ) + ) +} + +case class PopGeoTweetCandidatePredicates(override val config: Config) + extends OutOfNetworkTweetPredicates[OutOfNetworkTweetPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override def postCandidateSpecificPredicates: List[ + NamedPredicate[OutOfNetworkTweetPushCandidate] + ] = List( + PredicatesForCandidate.htlFatiguePredicate( + PushFeatureSwitchParams.NewUserPlaybookAllowedLastLoginHours + ) + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.scala new file mode 100644 index 0000000000..a4e8bec68a --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/F1FirstdegreeTweet.scala @@ -0,0 +1,60 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.F1FirstDegree +import com.twitter.frigate.common.base.SocialContextAction +import com.twitter.frigate.common.base.SocialGraphServiceRelationshipMap +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes._ +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.F1FirstDegreeTweetIbis2HydratorForCandidate +import com.twitter.frigate.pushservice.model.ntab.F1FirstDegreeTweetNTabRequestHydrator +import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPHWithoutSGSPredicates +import com.twitter.frigate.pushservice.util.CandidateHydrationUtil.TweetWithSocialContextTraits +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.stitch.tweetypie.TweetyPie +import com.twitter.util.Future + +class F1TweetPushCandidate( + candidate: RawCandidate with TweetWithSocialContextTraits, + author: Future[Option[User]], + socialGraphServiceResultMap: Map[RelationEdge, Boolean], + copyIds: CopyIds +)( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with F1FirstDegree + with TweetAuthorDetails + with SocialGraphServiceRelationshipMap + with F1FirstDegreeTweetNTabRequestHydrator + with F1FirstDegreeTweetIbis2HydratorForCandidate { + override val socialContextActions: Seq[SocialContextAction] = + candidate.socialContextActions + override val socialContextAllTypeActions: Seq[SocialContextAction] = + candidate.socialContextActions + override val statsReceiver: StatsReceiver = stats + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + override val tweetId: Long = candidate.tweetId + override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = + candidate.tweetyPieResult + override lazy val tweetAuthor: Future[Option[User]] = author + override val target: PushTypes.Target = candidate.target + override lazy val commonRecType: CommonRecommendationType = + candidate.commonRecType + override val pushCopyId: Option[Int] = copyIds.pushCopyId + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + override val copyAggregationId: Option[String] = copyIds.aggregationId + + override val relationshipMap: Map[RelationEdge, Boolean] = socialGraphServiceResultMap +} + +case class F1TweetCandidatePredicates(override val config: Config) + extends BasicTweetPredicatesForRFPHWithoutSGSPredicates[F1TweetPushCandidate] { + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.scala new file mode 100644 index 0000000000..412dfbf00d --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ListRecommendationPushCandidate.scala @@ -0,0 +1,72 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.channels.common.thriftscala.ApiList +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.ListPushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.ListIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.ListCandidateNTabRequestHydrator +import com.twitter.frigate.pushservice.predicate.ListPredicates +import com.twitter.frigate.pushservice.take.predicates.BasicRFPHPredicates +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +class ListRecommendationPushCandidate( + val apiListStore: ReadableStore[Long, ApiList], + candidate: RawCandidate with ListPushCandidate, + copyIds: CopyIds +)( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with ListPushCandidate + with ListIbis2Hydrator + with ListCandidateNTabRequestHydrator { + + override val commonRecType: CommonRecommendationType = candidate.commonRecType + + override val pushCopyId: Option[Int] = copyIds.pushCopyId + + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + + override val copyAggregationId: Option[String] = copyIds.aggregationId + + override val statsReceiver: StatsReceiver = stats + + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + + override val target: PushTypes.Target = candidate.target + + override val listId: Long = candidate.listId + + lazy val apiList: Future[Option[ApiList]] = apiListStore.get(listId) + + lazy val listName: Future[Option[String]] = apiList.map { apiListOpt => + apiListOpt.map(_.name) + } + + lazy val listOwnerId: Future[Option[Long]] = apiList.map { apiListOpt => + apiListOpt.map(_.ownerId) + } + +} + +case class ListRecommendationPredicates(config: Config) + extends BasicRFPHPredicates[ListRecommendationPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val predicates: List[NamedPredicate[ListRecommendationPushCandidate]] = List( + ListPredicates.listNameExistsPredicate(), + ListPredicates.listAuthorExistsPredicate(), + ListPredicates.listAuthorAcceptableToTargetUser(config.edgeStore), + ListPredicates.listAcceptablePredicate(), + ListPredicates.listSubscriberCountPredicate() + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.scala new file mode 100644 index 0000000000..65633259c9 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutCreatorEventPushCandidate.scala @@ -0,0 +1,136 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.HydratedMagicFanoutCreatorEventCandidate +import com.twitter.frigate.common.base.MagicFanoutCreatorEventCandidate +import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType +import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.MagicFanoutCreatorEventIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.MagicFanoutCreatorEventNtabRequestHydrator +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate +import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.storehaus.ReadableStore +import com.twitter.strato.client.UserId +import com.twitter.util.Future +import scala.util.control.NoStackTrace + +class MagicFanoutCreatorEventPushCandidateHydratorException(private val message: String) + extends Exception(message) + with NoStackTrace + +class MagicFanoutCreatorEventPushCandidate( + candidate: RawCandidate with MagicFanoutCreatorEventCandidate, + creatorUser: Option[User], + copyIds: CopyIds, + creatorTweetCountStore: ReadableStore[UserId, Int] +)( + implicit val statsScoped: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with MagicFanoutCreatorEventIbis2Hydrator + with MagicFanoutCreatorEventNtabRequestHydrator + with MagicFanoutCreatorEventCandidate + with HydratedMagicFanoutCreatorEventCandidate { + override def creatorId: Long = candidate.creatorId + + override def hydratedCreator: Option[User] = creatorUser + + override lazy val numberOfTweetsFut: Future[Option[Int]] = + creatorTweetCountStore.get(UserId(creatorId)) + + lazy val userProfile = hydratedCreator + .flatMap(_.profile).getOrElse( + throw new MagicFanoutCreatorEventPushCandidateHydratorException( + s"Unable to get user profile to generate tapThrough for userId: $creatorId")) + + override val frigateNotification: FrigateNotification = candidate.frigateNotification + + override def subscriberId: Option[Long] = candidate.subscriberId + + override def creatorFanoutType: CreatorFanoutType = candidate.creatorFanoutType + + override def target: PushTypes.Target = candidate.target + + override def pushId: Long = candidate.pushId + + override def candidateMagicEventsReasons: Seq[MagicEventsReason] = + candidate.candidateMagicEventsReasons + + override def statsReceiver: StatsReceiver = statsScoped + + override def pushCopyId: Option[Int] = copyIds.pushCopyId + + override def ntabCopyId: Option[Int] = copyIds.ntabCopyId + + override def copyAggregationId: Option[String] = copyIds.aggregationId + + override def commonRecType: CommonRecommendationType = candidate.commonRecType + + override def weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + +} + +case class MagicFanouCreatorSubscriptionEventPushPredicates(config: Config) + extends BasicSendHandlerPredicates[MagicFanoutCreatorEventPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val preCandidateSpecificPredicates: List[ + NamedPredicate[MagicFanoutCreatorEventPushCandidate] + ] = + List( + PredicatesForCandidate.paramPredicate( + PushFeatureSwitchParams.EnableCreatorSubscriptionPush + ), + PredicatesForCandidate.isDeviceEligibleForCreatorPush, + MagicFanoutPredicatesForCandidate.creatorPushTargetIsNotCreator(), + MagicFanoutPredicatesForCandidate.duplicateCreatorPredicate, + MagicFanoutPredicatesForCandidate.magicFanoutCreatorPushFatiguePredicate(), + ) + + override val postCandidateSpecificPredicates: List[ + NamedPredicate[MagicFanoutCreatorEventPushCandidate] + ] = + List( + MagicFanoutNtabCaretFatiguePredicate(), + MagicFanoutPredicatesForCandidate.isSuperFollowingCreator()(config, statsReceiver).flip + ) +} + +case class MagicFanoutNewCreatorEventPushPredicates(config: Config) + extends BasicSendHandlerPredicates[MagicFanoutCreatorEventPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val preCandidateSpecificPredicates: List[ + NamedPredicate[MagicFanoutCreatorEventPushCandidate] + ] = + List( + PredicatesForCandidate.paramPredicate( + PushFeatureSwitchParams.EnableNewCreatorPush + ), + PredicatesForCandidate.isDeviceEligibleForCreatorPush, + MagicFanoutPredicatesForCandidate.duplicateCreatorPredicate, + MagicFanoutPredicatesForCandidate.magicFanoutCreatorPushFatiguePredicate, + ) + + override val postCandidateSpecificPredicates: List[ + NamedPredicate[MagicFanoutCreatorEventPushCandidate] + ] = + List( + MagicFanoutNtabCaretFatiguePredicate(), + MagicFanoutPredicatesForCandidate.isSuperFollowingCreator()(config, statsReceiver).flip + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.scala new file mode 100644 index 0000000000..e0a5f53866 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutEventPushCandidate.scala @@ -0,0 +1,303 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.escherbird.metadata.thriftscala.EntityMegadata +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.MagicFanoutEventCandidate +import com.twitter.frigate.common.base.RecommendationType +import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext +import com.twitter.frigate.common.util.HighPriorityLocaleUtil +import com.twitter.frigate.magic_events.thriftscala.FanoutEvent +import com.twitter.frigate.magic_events.thriftscala.FanoutMetadata +import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason +import com.twitter.frigate.magic_events.thriftscala.NewsForYouMetadata +import com.twitter.frigate.magic_events.thriftscala.ReasonSource +import com.twitter.frigate.magic_events.thriftscala.TargetID +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.Ibis2HydratorForCandidate +import com.twitter.frigate.pushservice.model.ntab.EventNTabRequestHydrator +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil +import com.twitter.frigate.pushservice.store.EventRequest +import com.twitter.frigate.pushservice.store.UttEntityHydrationStore +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.pushservice.util.TopicsUtil +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails +import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery +import com.twitter.interests.thriftscala.InterestId.SemanticCore +import com.twitter.interests.thriftscala.UserInterests +import com.twitter.livevideo.common.ids.CountryId +import com.twitter.livevideo.common.ids.UserId +import com.twitter.livevideo.timeline.domain.v2.Event +import com.twitter.livevideo.timeline.domain.v2.HydrationOptions +import com.twitter.livevideo.timeline.domain.v2.LookupContext +import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities +import com.twitter.storehaus.ReadableStore +import com.twitter.topiclisting.utt.LocalizedEntity +import com.twitter.util.Future + +abstract class MagicFanoutEventPushCandidate( + candidate: RawCandidate with MagicFanoutEventCandidate with RecommendationType, + copyIds: CopyIds, + override val fanoutEvent: Option[FanoutEvent], + override val semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]], + simClusterToEntities: Map[Int, Option[SimClustersInferredEntities]], + lexServiceStore: ReadableStore[EventRequest, Event], + interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], + uttEntityHydrationStore: UttEntityHydrationStore +)( + implicit statsScoped: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with MagicFanoutEventHydratedCandidate + with MagicFanoutEventCandidate + with EventNTabRequestHydrator + with RecommendationType + with Ibis2HydratorForCandidate { + + override lazy val eventFut: Future[Option[Event]] = { + eventRequestFut.flatMap { + case Some(eventRequest) => lexServiceStore.get(eventRequest) + case _ => Future.None + } + } + + override val frigateNotification: FrigateNotification = candidate.frigateNotification + + override val pushId: Long = candidate.pushId + + override val candidateMagicEventsReasons: Seq[MagicEventsReason] = + candidate.candidateMagicEventsReasons + + override val eventId: Long = candidate.eventId + + override val momentId: Option[Long] = candidate.momentId + + override val target: Target = candidate.target + + override val eventLanguage: Option[String] = candidate.eventLanguage + + override val details: Option[MagicFanoutEventNotificationDetails] = candidate.details + + override lazy val stats: StatsReceiver = statsScoped.scope("MagicFanoutEventPushCandidate") + + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + + override val pushCopyId: Option[Int] = copyIds.pushCopyId + + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + + override val copyAggregationId: Option[String] = copyIds.aggregationId + + override val statsReceiver: StatsReceiver = statsScoped.scope("MagicFanoutEventPushCandidate") + + override val effectiveMagicEventsReasons: Option[Seq[MagicEventsReason]] = Some( + candidateMagicEventsReasons) + + lazy val newsForYouMetadata: Option[NewsForYouMetadata] = + fanoutEvent.flatMap { event => + { + event.fanoutMetadata.collect { + case FanoutMetadata.NewsForYouMetadata(nfyMetadata) => nfyMetadata + } + } + } + + val reverseIndexedTopicIds = candidate.candidateMagicEventsReasons + .filter(_.source.contains(ReasonSource.UttTopicFollowGraph)) + .map(_.reason).collect { + case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.entityId + }.toSet + + val ergSemanticCoreIds = candidate.candidateMagicEventsReasons + .filter(_.source.contains(ReasonSource.ErgShortTermInterestSemanticCore)).map( + _.reason).collect { + case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.entityId + }.toSet + + override lazy val ergLocalizedEntities = TopicsUtil + .getLocalizedEntityMap(target, ergSemanticCoreIds, uttEntityHydrationStore) + .map { localizedEntityMap => + ergSemanticCoreIds.collect { + case topicId if localizedEntityMap.contains(topicId) => localizedEntityMap(topicId) + } + } + + val eventSemanticCoreEntityIds: Seq[Long] = { + val entityIds = for { + event <- fanoutEvent + targets <- event.targets + } yield { + targets.flatMap { + _.whitelist.map { + _.collect { + case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.entityId + } + } + } + } + + entityIds.map(_.flatten).getOrElse(Seq.empty) + } + + val eventSemanticCoreDomainIds: Seq[Long] = { + val domainIds = for { + event <- fanoutEvent + targets <- event.targets + } yield { + targets.flatMap { + _.whitelist.map { + _.collect { + case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID.domainId + } + } + } + } + + domainIds.map(_.flatten).getOrElse(Seq.empty) + } + + override lazy val followedTopicLocalizedEntities: Future[Set[LocalizedEntity]] = { + + val isNewSignupTargetingReason = candidateMagicEventsReasons.size == 1 && + candidateMagicEventsReasons.headOption.exists(_.source.contains(ReasonSource.NewSignup)) + + val shouldFetchTopicFollows = reverseIndexedTopicIds.nonEmpty || isNewSignupTargetingReason + + val topicFollows = if (shouldFetchTopicFollows) { + TopicsUtil + .getTopicsFollowedByUser( + candidate.target, + interestsLookupStore, + stats.stat("followed_topics") + ).map { _.getOrElse(Seq.empty) }.map { + _.flatMap { + _.interestId match { + case SemanticCore(semanticCore) => Some(semanticCore.id) + case _ => None + } + } + } + } else Future.Nil + + topicFollows.flatMap { followedTopicIds => + val topicIds = if (isNewSignupTargetingReason) { + // if new signup is the only targeting reason then we check the event targeting reason + // against realtime topic follows. + eventSemanticCoreEntityIds.toSet.intersect(followedTopicIds.toSet) + } else { + // check against the fanout reason of topics + followedTopicIds.toSet.intersect(reverseIndexedTopicIds) + } + + TopicsUtil + .getLocalizedEntityMap(target, topicIds, uttEntityHydrationStore) + .map { localizedEntityMap => + topicIds.collect { + case topicId if localizedEntityMap.contains(topicId) => localizedEntityMap(topicId) + } + } + } + } + + lazy val simClusterToEntityMapping: Map[Int, Seq[Long]] = + simClusterToEntities.flatMap { + case (clusterId, Some(inferredEntities)) => + statsReceiver.counter("with_cluster_to_entity_mapping").incr() + Some( + ( + clusterId, + inferredEntities.entities + .map(_.entityId))) + case _ => + statsReceiver.counter("without_cluster_to_entity_mapping").incr() + None + } + + lazy val annotatedAndInferredSemanticCoreEntities: Seq[Long] = + (simClusterToEntityMapping, eventFanoutReasonEntities) match { + case (entityMapping, eventFanoutReasons) => + entityMapping.values.flatten.toSeq ++ + eventFanoutReasons.semanticCoreIds.map(_.entityId) + } + + lazy val shouldHydrateSquareImage = target.deviceInfo.map { deviceInfo => + (PushDeviceUtil.isPrimaryDeviceIOS(deviceInfo) && + target.params(PushFeatureSwitchParams.EnableEventSquareMediaIosMagicFanoutNewsEvent)) || + (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfo) && + target.params(PushFeatureSwitchParams.EnableEventSquareMediaAndroid)) + } + + lazy val shouldHydratePrimaryImage: Future[Boolean] = target.deviceInfo.map { deviceInfo => + (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfo) && + target.params(PushFeatureSwitchParams.EnableEventPrimaryMediaAndroid)) + } + + lazy val eventRequestFut: Future[Option[EventRequest]] = + Future + .join( + target.inferredUserDeviceLanguage, + target.accountCountryCode, + shouldHydrateSquareImage, + shouldHydratePrimaryImage).map { + case ( + inferredUserDeviceLanguage, + accountCountryCode, + shouldHydrateSquareImage, + shouldHydratePrimaryImage) => + if (shouldHydrateSquareImage || shouldHydratePrimaryImage) { + Some( + EventRequest( + eventId, + lookupContext = LookupContext( + hydrationOptions = HydrationOptions( + includeSquareImage = shouldHydrateSquareImage, + includePrimaryImage = shouldHydratePrimaryImage + ), + language = inferredUserDeviceLanguage, + countryCode = accountCountryCode, + userId = Some(UserId(target.targetId)) + ) + )) + } else { + Some( + EventRequest( + eventId, + lookupContext = LookupContext( + language = inferredUserDeviceLanguage, + countryCode = accountCountryCode + ) + )) + } + case _ => None + } + + lazy val isHighPriorityEvent: Future[Boolean] = target.accountCountryCode.map { countryCodeOpt => + val isHighPriorityPushOpt = for { + countryCode <- countryCodeOpt + nfyMetadata <- newsForYouMetadata + eventContext <- nfyMetadata.eventContextScribe + } yield { + val highPriorityLocales = HighPriorityLocaleUtil.getHighPriorityLocales( + eventContext = eventContext, + defaultLocalesOpt = nfyMetadata.locales) + val highPriorityGeos = HighPriorityLocaleUtil.getHighPriorityGeos( + eventContext = eventContext, + defaultGeoPlaceIdsOpt = nfyMetadata.placeIds) + val isHighPriorityLocalePush = + highPriorityLocales.flatMap(_.country).map(CountryId(_)).contains(CountryId(countryCode)) + val isHighPriorityGeoPush = MagicFanoutPredicatesUtil + .geoPlaceIdsFromReasons(candidateMagicEventsReasons) + .intersect(highPriorityGeos.toSet) + .nonEmpty + stats.scope("is_high_priority_locale_push").counter(s"$isHighPriorityLocalePush").incr() + stats.scope("is_high_priority_geo_push").counter(s"$isHighPriorityGeoPush").incr() + isHighPriorityLocalePush || isHighPriorityGeoPush + } + isHighPriorityPushOpt.getOrElse(false) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.scala new file mode 100644 index 0000000000..36196120bb --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutHydratedCandidate.scala @@ -0,0 +1,147 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.escherbird.common.thriftscala.QualifiedId +import com.twitter.escherbird.metadata.thriftscala.BasicMetadata +import com.twitter.escherbird.metadata.thriftscala.EntityIndexFields +import com.twitter.escherbird.metadata.thriftscala.EntityMegadata +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.MagicFanoutCandidate +import com.twitter.frigate.common.base.MagicFanoutEventCandidate +import com.twitter.frigate.common.base.RichEventFutCandidate +import com.twitter.frigate.magic_events.thriftscala +import com.twitter.frigate.magic_events.thriftscala.AnnotationAlg +import com.twitter.frigate.magic_events.thriftscala.FanoutEvent +import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason +import com.twitter.frigate.magic_events.thriftscala.SemanticCoreID +import com.twitter.frigate.magic_events.thriftscala.SimClusterID +import com.twitter.frigate.magic_events.thriftscala.TargetID +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery +import com.twitter.livevideo.timeline.domain.v2.Event +import com.twitter.topiclisting.utt.LocalizedEntity +import com.twitter.util.Future + +case class FanoutReasonEntities( + userIds: Set[Long], + placeIds: Set[Long], + semanticCoreIds: Set[SemanticCoreID], + simclusterIds: Set[SimClusterID]) { + val qualifiedIds: Set[QualifiedId] = + semanticCoreIds.map(e => QualifiedId(e.domainId, e.entityId)) +} + +object FanoutReasonEntities { + val empty = FanoutReasonEntities( + userIds = Set.empty, + placeIds = Set.empty, + semanticCoreIds = Set.empty, + simclusterIds = Set.empty + ) + + def from(reasons: Seq[TargetID]): FanoutReasonEntities = { + val userIds: Set[Long] = reasons.collect { + case TargetID.UserID(userId) => userId.id + }.toSet + val placeIds: Set[Long] = reasons.collect { + case TargetID.PlaceID(placeId) => placeId.id + }.toSet + val semanticCoreIds: Set[SemanticCoreID] = reasons.collect { + case TargetID.SemanticCoreID(semanticCoreID) => semanticCoreID + }.toSet + val simclusterIds: Set[SimClusterID] = reasons.collect { + case TargetID.SimClusterID(simClusterID) => simClusterID + }.toSet + + FanoutReasonEntities( + userIds = userIds, + placeIds, + semanticCoreIds = semanticCoreIds, + simclusterIds = simclusterIds + ) + } +} + +trait MagicFanoutHydratedCandidate extends PushCandidate with MagicFanoutCandidate { + lazy val fanoutReasonEntities: FanoutReasonEntities = + FanoutReasonEntities.from(candidateMagicEventsReasons.map(_.reason)) +} + +trait MagicFanoutEventHydratedCandidate + extends MagicFanoutHydratedCandidate + with MagicFanoutEventCandidate + with RichEventFutCandidate { + + def target: PushTypes.Target + + def stats: StatsReceiver + + def fanoutEvent: Option[FanoutEvent] + + def eventFut: Future[Option[Event]] + + def semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]] + + def effectiveMagicEventsReasons: Option[Seq[MagicEventsReason]] + + def followedTopicLocalizedEntities: Future[Set[LocalizedEntity]] + + def ergLocalizedEntities: Future[Set[LocalizedEntity]] + + lazy val entityAnnotationAlg: Map[TargetID, Set[AnnotationAlg]] = + fanoutEvent + .flatMap { metadata => + metadata.eventAnnotationInfo.map { eventAnnotationInfo => + eventAnnotationInfo.map { + case (target, annotationInfoSet) => target -> annotationInfoSet.map(_.alg).toSet + }.toMap + } + }.getOrElse(Map.empty) + + lazy val eventSource: Option[String] = fanoutEvent.map { metadata => + val source = metadata.eventSource.getOrElse("undefined") + stats.scope("eventSource").counter(source).incr() + source + } + + lazy val semanticCoreEntityTags: Map[(Long, Long), Set[String]] = + semanticEntityResults.flatMap { + case (semanticEntityForQuery, entityMegadataOpt: Option[EntityMegadata]) => + for { + entityMegadata <- entityMegadataOpt + basicMetadata: BasicMetadata <- entityMegadata.basicMetadata + indexableFields: EntityIndexFields <- basicMetadata.indexableFields + tags <- indexableFields.tags + } yield { + ((semanticEntityForQuery.domainId, semanticEntityForQuery.entityId), tags.toSet) + } + } + + lazy val owningTwitterUserIds: Seq[Long] = semanticEntityResults.values.flatten + .flatMap { + _.basicMetadata.flatMap(_.twitter.flatMap(_.owningTwitterUserIds)) + }.flatten + .toSeq + .distinct + + lazy val eventFanoutReasonEntities: FanoutReasonEntities = + fanoutEvent match { + case Some(fanout) => + fanout.targets + .map { targets: Seq[thriftscala.Target] => + FanoutReasonEntities.from(targets.flatMap(_.whitelist).flatten) + }.getOrElse(FanoutReasonEntities.empty) + case _ => FanoutReasonEntities.empty + } + + override lazy val eventResultFut: Future[Event] = eventFut.map { + case Some(eventResult) => eventResult + case _ => + throw new IllegalArgumentException("event is None for MagicFanoutEventHydratedCandidate") + } + override val rankScore: Option[Double] = None + override val predictionScore: Option[Double] = None +} + +case class MagicFanoutEventHydratedInfo( + fanoutEvent: Option[FanoutEvent], + semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]]) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.scala new file mode 100644 index 0000000000..61ead1f22e --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutNewsEvent.scala @@ -0,0 +1,99 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.escherbird.metadata.thriftscala.EntityMegadata +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.MagicFanoutNewsEventCandidate +import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext +import com.twitter.frigate.magic_events.thriftscala.FanoutEvent +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.MagicFanoutNewsEventIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.MagicFanoutNewsEventNTabRequestHydrator +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.event.EventPredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutTargetingPredicateWrappersForCandidate +import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate +import com.twitter.frigate.pushservice.store.EventRequest +import com.twitter.frigate.pushservice.store.UttEntityHydrationStore +import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery +import com.twitter.interests.thriftscala.UserInterests +import com.twitter.livevideo.timeline.domain.v2.Event +import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities +import com.twitter.storehaus.ReadableStore + +class MagicFanoutNewsEventPushCandidate( + candidate: RawCandidate with MagicFanoutNewsEventCandidate, + copyIds: CopyIds, + override val fanoutEvent: Option[FanoutEvent], + override val semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]], + simClusterToEntities: Map[Int, Option[SimClustersInferredEntities]], + lexServiceStore: ReadableStore[EventRequest, Event], + interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], + uttEntityHydrationStore: UttEntityHydrationStore +)( + implicit statsScoped: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends MagicFanoutEventPushCandidate( + candidate, + copyIds, + fanoutEvent, + semanticEntityResults, + simClusterToEntities, + lexServiceStore, + interestsLookupStore, + uttEntityHydrationStore + )(statsScoped, pushModelScorer) + with MagicFanoutNewsEventCandidate + with MagicFanoutNewsEventIbis2Hydrator + with MagicFanoutNewsEventNTabRequestHydrator { + + override lazy val stats: StatsReceiver = statsScoped.scope("MagicFanoutNewsEventPushCandidate") + override val statsReceiver: StatsReceiver = statsScoped.scope("MagicFanoutNewsEventPushCandidate") +} + +case class MagicFanoutNewsEventCandidatePredicates(config: Config) + extends BasicSendHandlerPredicates[MagicFanoutNewsEventPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val preCandidateSpecificPredicates: List[ + NamedPredicate[MagicFanoutNewsEventPushCandidate] + ] = + List( + EventPredicatesForCandidate.accountCountryPredicateWithAllowlist, + PredicatesForCandidate.isDeviceEligibleForNewsOrSports, + MagicFanoutPredicatesForCandidate.inferredUserDeviceLanguagePredicate, + PredicatesForCandidate.secondaryDormantAccountPredicate(statsReceiver), + MagicFanoutPredicatesForCandidate.highPriorityNewsEventExceptedPredicate( + MagicFanoutTargetingPredicateWrappersForCandidate + .magicFanoutTargetingPredicate(statsReceiver, config) + )(config), + MagicFanoutPredicatesForCandidate.geoOptOutPredicate(config.safeUserStore), + EventPredicatesForCandidate.isNotDuplicateWithEventId, + MagicFanoutPredicatesForCandidate.highPriorityNewsEventExceptedPredicate( + MagicFanoutPredicatesForCandidate.newsNotificationFatigue() + )(config), + MagicFanoutPredicatesForCandidate.highPriorityNewsEventExceptedPredicate( + MagicFanoutNtabCaretFatiguePredicate() + )(config), + MagicFanoutPredicatesForCandidate.escherbirdMagicfanoutEventParam()(statsReceiver), + MagicFanoutPredicatesForCandidate.hasCustomTargetingForNewsEventsParam( + statsReceiver + ) + ) + + override val postCandidateSpecificPredicates: List[ + NamedPredicate[MagicFanoutNewsEventPushCandidate] + ] = + List( + MagicFanoutPredicatesForCandidate.magicFanoutNoOptoutInterestPredicate, + MagicFanoutPredicatesForCandidate.geoTargetingHoldback(), + MagicFanoutPredicatesForCandidate.userGeneratedEventsPredicate, + EventPredicatesForCandidate.hasTitle, + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.scala new file mode 100644 index 0000000000..4dc569e2b3 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutProductLaunchPushCandidate.scala @@ -0,0 +1,95 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate +import com.twitter.frigate.common.util.{FeatureSwitchParams => FS} +import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason +import com.twitter.frigate.magic_events.thriftscala.ProductType +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.MagicFanoutProductLaunchIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.MagicFanoutProductLaunchNtabRequestHydrator +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate +import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.hermit.predicate.NamedPredicate + +class MagicFanoutProductLaunchPushCandidate( + candidate: RawCandidate with MagicFanoutProductLaunchCandidate, + copyIds: CopyIds +)( + implicit val statsScoped: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with MagicFanoutProductLaunchCandidate + with MagicFanoutProductLaunchIbis2Hydrator + with MagicFanoutProductLaunchNtabRequestHydrator { + + override val frigateNotification: FrigateNotification = candidate.frigateNotification + + override val pushCopyId: Option[Int] = copyIds.pushCopyId + + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + + override val pushId: Long = candidate.pushId + + override val productLaunchType: ProductType = candidate.productLaunchType + + override val candidateMagicEventsReasons: Seq[MagicEventsReason] = + candidate.candidateMagicEventsReasons + + override val copyAggregationId: Option[String] = copyIds.aggregationId + + override val target: Target = candidate.target + + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + + override val statsReceiver: StatsReceiver = + statsScoped.scope("MagicFanoutProductLaunchPushCandidate") +} + +case class MagicFanoutProductLaunchPushCandidatePredicates(config: Config) + extends BasicSendHandlerPredicates[MagicFanoutProductLaunchPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val preCandidateSpecificPredicates: List[ + NamedPredicate[MagicFanoutProductLaunchPushCandidate] + ] = + List( + PredicatesForCandidate.isDeviceEligibleForCreatorPush, + PredicatesForCandidate.exceptedPredicate( + "excepted_is_target_blue_verified", + MagicFanoutPredicatesUtil.shouldSkipBlueVerifiedCheckForCandidate, + PredicatesForCandidate.isTargetBlueVerified.flip + ), // no need to send if target is already Blue Verified + PredicatesForCandidate.exceptedPredicate( + "excepted_is_target_legacy_verified", + MagicFanoutPredicatesUtil.shouldSkipLegacyVerifiedCheckForCandidate, + PredicatesForCandidate.isTargetLegacyVerified.flip + ), // no need to send if target is already Legacy Verified + PredicatesForCandidate.exceptedPredicate( + "excepted_is_target_super_follow_creator", + MagicFanoutPredicatesUtil.shouldSkipSuperFollowCreatorCheckForCandidate, + PredicatesForCandidate.isTargetSuperFollowCreator.flip + ), // no need to send if target is already Super Follow Creator + PredicatesForCandidate.paramPredicate( + FS.EnableMagicFanoutProductLaunch + ), + MagicFanoutPredicatesForCandidate.magicFanoutProductLaunchFatigue(), + ) + + override val postCandidateSpecificPredicates: List[ + NamedPredicate[MagicFanoutProductLaunchPushCandidate] + ] = + List( + MagicFanoutNtabCaretFatiguePredicate(), + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.scala new file mode 100644 index 0000000000..84535e4c24 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/MagicFanoutSportsPushCandidate.scala @@ -0,0 +1,119 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.escherbird.metadata.thriftscala.EntityMegadata +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.BaseGameScore +import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate +import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation +import com.twitter.frigate.common.base.TeamInfo +import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext +import com.twitter.frigate.magic_events.thriftscala.FanoutEvent +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.MagicFanoutSportsEventIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.MagicFanoutSportsEventNTabRequestHydrator +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutTargetingPredicateWrappersForCandidate +import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.MagicFanoutNtabCaretFatiguePredicate +import com.twitter.frigate.pushservice.store.EventRequest +import com.twitter.frigate.pushservice.store.UttEntityHydrationStore +import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery +import com.twitter.interests.thriftscala.UserInterests +import com.twitter.livevideo.timeline.domain.v2.Event +import com.twitter.livevideo.timeline.domain.v2.HydrationOptions +import com.twitter.livevideo.timeline.domain.v2.LookupContext +import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +class MagicFanoutSportsPushCandidate( + candidate: RawCandidate + with MagicFanoutSportsEventCandidate + with MagicFanoutSportsScoreInformation, + copyIds: CopyIds, + override val fanoutEvent: Option[FanoutEvent], + override val semanticEntityResults: Map[SemanticEntityForQuery, Option[EntityMegadata]], + simClusterToEntities: Map[Int, Option[SimClustersInferredEntities]], + lexServiceStore: ReadableStore[EventRequest, Event], + interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], + uttEntityHydrationStore: UttEntityHydrationStore +)( + implicit statsScoped: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends MagicFanoutEventPushCandidate( + candidate, + copyIds, + fanoutEvent, + semanticEntityResults, + simClusterToEntities, + lexServiceStore, + interestsLookupStore, + uttEntityHydrationStore)(statsScoped, pushModelScorer) + with MagicFanoutSportsEventCandidate + with MagicFanoutSportsScoreInformation + with MagicFanoutSportsEventNTabRequestHydrator + with MagicFanoutSportsEventIbis2Hydrator { + + override val isScoreUpdate: Boolean = candidate.isScoreUpdate + override val gameScores: Future[Option[BaseGameScore]] = candidate.gameScores + override val homeTeamInfo: Future[Option[TeamInfo]] = candidate.homeTeamInfo + override val awayTeamInfo: Future[Option[TeamInfo]] = candidate.awayTeamInfo + + override lazy val stats: StatsReceiver = statsScoped.scope("MagicFanoutSportsPushCandidate") + override val statsReceiver: StatsReceiver = statsScoped.scope("MagicFanoutSportsPushCandidate") + + override lazy val eventRequestFut: Future[Option[EventRequest]] = { + Future.join(target.inferredUserDeviceLanguage, target.accountCountryCode).map { + case (inferredUserDeviceLanguage, accountCountryCode) => + Some( + EventRequest( + eventId, + lookupContext = LookupContext( + hydrationOptions = HydrationOptions( + includeSquareImage = true, + includePrimaryImage = true + ), + language = inferredUserDeviceLanguage, + countryCode = accountCountryCode + ) + )) + } + } +} + +case class MagicFanoutSportsEventCandidatePredicates(config: Config) + extends BasicSendHandlerPredicates[MagicFanoutSportsPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val preCandidateSpecificPredicates: List[ + NamedPredicate[MagicFanoutSportsPushCandidate] + ] = + List( + PredicatesForCandidate.paramPredicate(PushFeatureSwitchParams.EnableScoreFanoutNotification) + ) + + override val postCandidateSpecificPredicates: List[ + NamedPredicate[MagicFanoutSportsPushCandidate] + ] = + List( + PredicatesForCandidate.isDeviceEligibleForNewsOrSports, + MagicFanoutPredicatesForCandidate.inferredUserDeviceLanguagePredicate, + MagicFanoutPredicatesForCandidate.highPriorityEventExceptedPredicate( + MagicFanoutTargetingPredicateWrappersForCandidate + .magicFanoutTargetingPredicate(statsReceiver, config) + )(config), + PredicatesForCandidate.secondaryDormantAccountPredicate( + statsReceiver + ), + MagicFanoutPredicatesForCandidate.highPriorityEventExceptedPredicate( + MagicFanoutNtabCaretFatiguePredicate() + )(config), + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.scala new file mode 100644 index 0000000000..0b8c533eaf --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/OutOfNetworkTweetPushCandidate.scala @@ -0,0 +1,68 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.contentrecommender.thriftscala.MetricTag +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate +import com.twitter.frigate.common.base.TopicCandidate +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.OutOfNetworkTweetIbis2HydratorForCandidate +import com.twitter.frigate.pushservice.model.ntab.OutOfNetworkTweetNTabRequestHydrator +import com.twitter.frigate.pushservice.predicate.HealthPredicates +import com.twitter.frigate.pushservice.take.predicates.OutOfNetworkTweetPredicates +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.stitch.tweetypie.TweetyPie +import com.twitter.topiclisting.utt.LocalizedEntity +import com.twitter.util.Future + +class OutOfNetworkTweetPushCandidate( + candidate: RawCandidate with OutOfNetworkTweetCandidate with TopicCandidate, + author: Future[Option[User]], + copyIds: CopyIds +)( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with OutOfNetworkTweetCandidate + with TopicCandidate + with TweetAuthorDetails + with OutOfNetworkTweetNTabRequestHydrator + with OutOfNetworkTweetIbis2HydratorForCandidate { + override val statsReceiver: StatsReceiver = stats + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + override val tweetId: Long = candidate.tweetId + override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = + candidate.tweetyPieResult + override lazy val tweetAuthor: Future[Option[User]] = author + override val target: PushTypes.Target = candidate.target + override lazy val commonRecType: CommonRecommendationType = + candidate.commonRecType + override val pushCopyId: Option[Int] = copyIds.pushCopyId + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + override val copyAggregationId: Option[String] = copyIds.aggregationId + override lazy val semanticCoreEntityId: Option[Long] = candidate.semanticCoreEntityId + override lazy val localizedUttEntity: Option[LocalizedEntity] = candidate.localizedUttEntity + override lazy val algorithmCR: Option[String] = candidate.algorithmCR + override lazy val isMrBackfillCR: Option[Boolean] = candidate.isMrBackfillCR + override lazy val tagsCR: Option[Seq[MetricTag]] = candidate.tagsCR +} + +case class OutOfNetworkTweetCandidatePredicates(override val config: Config) + extends OutOfNetworkTweetPredicates[OutOfNetworkTweetPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override def postCandidateSpecificPredicates: List[ + NamedPredicate[OutOfNetworkTweetPushCandidate] + ] = + List( + HealthPredicates.agathaAbusiveTweetAuthorPredicateMrTwistly(), + ) + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.scala new file mode 100644 index 0000000000..83d5b67c31 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/PushTypes.scala @@ -0,0 +1,61 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.candidate.UserLanguage +import com.twitter.frigate.common.candidate._ +import com.twitter.frigate.data_pipeline.features_common.RequestContextForFeatureStore +import com.twitter.frigate.pushservice.model.candidate.CopyInfo +import com.twitter.frigate.pushservice.model.candidate.MLScores +import com.twitter.frigate.pushservice.model.candidate.QualityScribing +import com.twitter.frigate.pushservice.model.candidate.Scriber +import com.twitter.frigate.pushservice.model.ibis.Ibis2HydratorForCandidate +import com.twitter.frigate.pushservice.model.ntab.NTabRequest +import com.twitter.frigate.pushservice.take.ChannelForCandidate +import com.twitter.frigate.pushservice.target._ +import com.twitter.util.Time + +object PushTypes { + + trait Target + extends TargetUser + with UserDetails + with TargetWithPushContext + with TargetDecider + with TargetABDecider + with FrigateHistory + with PushTargeting + with TargetScoringDetails + with TweetImpressionHistory + with CustomConfigForExpt + with CaretFeedbackHistory + with NotificationFeedbackHistory + with PromptFeedbackHistory + with HTLVisitHistory + with MaxTweetAge + with NewUserDetails + with ResurrectedUserDetails + with TargetWithSeedUsers + with MagicFanoutHistory + with OptOutUserInterests + with RequestContextForFeatureStore + with TargetAppPermissions + with UserLanguage + with InlineActionHistory + with TargetPlaces + + trait RawCandidate extends Candidate with TargetInfo[PushTypes.Target] with RecommendationType { + + val createdAt: Time = Time.now + } + + trait PushCandidate + extends RawCandidate + with CandidateScoringDetails + with MLScores + with QualityScribing + with CopyInfo + with Scriber + with Ibis2HydratorForCandidate + with NTabRequest + with ChannelForCandidate +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.scala new file mode 100644 index 0000000000..6da10ed77a --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSpeaker.scala @@ -0,0 +1,85 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.ScheduledSpaceSpeakerCandidate +import com.twitter.frigate.common.base.SpaceCandidateFanoutDetails +import com.twitter.frigate.common.util.FeatureSwitchParams +import com.twitter.frigate.magic_events.thriftscala.SpaceMetadata +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.ScheduledSpaceSpeakerIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.ScheduledSpaceSpeakerNTabRequestHydrator +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.SpacePredicate +import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.storehaus.ReadableStore +import com.twitter.ubs.thriftscala.AudioSpace +import com.twitter.util.Future + +class ScheduledSpaceSpeakerPushCandidate( + candidate: RawCandidate with ScheduledSpaceSpeakerCandidate, + hostUser: Option[User], + copyIds: CopyIds, + audioSpaceStore: ReadableStore[String, AudioSpace] +)( + implicit val statsScoped: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with ScheduledSpaceSpeakerCandidate + with ScheduledSpaceSpeakerIbis2Hydrator + with SpaceCandidateFanoutDetails + with ScheduledSpaceSpeakerNTabRequestHydrator { + + override val startTime: Long = candidate.startTime + + override val hydratedHost: Option[User] = hostUser + + override val spaceId: String = candidate.spaceId + + override val hostId: Option[Long] = candidate.hostId + + override val speakerIds: Option[Seq[Long]] = candidate.speakerIds + + override val listenerIds: Option[Seq[Long]] = candidate.listenerIds + + override val frigateNotification: FrigateNotification = candidate.frigateNotification + + override val pushCopyId: Option[Int] = copyIds.pushCopyId + + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + + override val copyAggregationId: Option[String] = copyIds.aggregationId + + override val target: Target = candidate.target + + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + + override lazy val audioSpaceFut: Future[Option[AudioSpace]] = audioSpaceStore.get(spaceId) + + override val spaceFanoutMetadata: Option[SpaceMetadata] = None + + override val statsReceiver: StatsReceiver = + statsScoped.scope("ScheduledSpaceSpeakerCandidate") +} + +case class ScheduledSpaceSpeakerCandidatePredicates(config: Config) + extends BasicSendHandlerPredicates[ScheduledSpaceSpeakerPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val preCandidateSpecificPredicates: List[ + NamedPredicate[ScheduledSpaceSpeakerPushCandidate] + ] = List( + SpacePredicate.scheduledSpaceStarted( + config.audioSpaceStore + ), + PredicatesForCandidate.paramPredicate(FeatureSwitchParams.EnableScheduledSpaceSpeakers) + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.scala new file mode 100644 index 0000000000..78977ab5d2 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ScheduledSpaceSubscriber.scala @@ -0,0 +1,86 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.ScheduledSpaceSubscriberCandidate +import com.twitter.frigate.common.base.SpaceCandidateFanoutDetails +import com.twitter.frigate.common.util.FeatureSwitchParams +import com.twitter.frigate.magic_events.thriftscala.SpaceMetadata +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.ScheduledSpaceSubscriberIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.ScheduledSpaceSubscriberNTabRequestHydrator +import com.twitter.frigate.pushservice.predicate._ +import com.twitter.frigate.pushservice.take.predicates.BasicSendHandlerPredicates +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.storehaus.ReadableStore +import com.twitter.ubs.thriftscala.AudioSpace +import com.twitter.util.Future + +class ScheduledSpaceSubscriberPushCandidate( + candidate: RawCandidate with ScheduledSpaceSubscriberCandidate, + hostUser: Option[User], + copyIds: CopyIds, + audioSpaceStore: ReadableStore[String, AudioSpace] +)( + implicit val statsScoped: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with ScheduledSpaceSubscriberCandidate + with SpaceCandidateFanoutDetails + with ScheduledSpaceSubscriberIbis2Hydrator + with ScheduledSpaceSubscriberNTabRequestHydrator { + + override val startTime: Long = candidate.startTime + + override val hydratedHost: Option[User] = hostUser + + override val spaceId: String = candidate.spaceId + + override val hostId: Option[Long] = candidate.hostId + + override val speakerIds: Option[Seq[Long]] = candidate.speakerIds + + override val listenerIds: Option[Seq[Long]] = candidate.listenerIds + + override val frigateNotification: FrigateNotification = candidate.frigateNotification + + override val pushCopyId: Option[Int] = copyIds.pushCopyId + + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + + override val copyAggregationId: Option[String] = copyIds.aggregationId + + override val target: Target = candidate.target + + override lazy val audioSpaceFut: Future[Option[AudioSpace]] = audioSpaceStore.get(spaceId) + + override val spaceFanoutMetadata: Option[SpaceMetadata] = None + + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + + override val statsReceiver: StatsReceiver = + statsScoped.scope("ScheduledSpaceSubscriberCandidate") +} + +case class ScheduledSpaceSubscriberCandidatePredicates(config: Config) + extends BasicSendHandlerPredicates[ScheduledSpaceSubscriberPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val preCandidateSpecificPredicates: List[ + NamedPredicate[ScheduledSpaceSubscriberPushCandidate] + ] = + List( + PredicatesForCandidate.paramPredicate(FeatureSwitchParams.EnableScheduledSpaceSubscribers), + SpacePredicate.narrowCastSpace, + SpacePredicate.targetInSpace(config.audioSpaceParticipantsStore), + SpacePredicate.spaceHostTargetUserBlocking(config.edgeStore), + PredicatesForCandidate.duplicateSpacesPredicate + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.scala new file mode 100644 index 0000000000..4d71a0c751 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/SubscribedSearchTweetPushCandidate.scala @@ -0,0 +1,56 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.SubscribedSearchTweetCandidate +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.SubscribedSearchTweetIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.SubscribedSearchTweetNtabRequestHydrator +import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.stitch.tweetypie.TweetyPie +import com.twitter.util.Future + +class SubscribedSearchTweetPushCandidate( + candidate: RawCandidate with SubscribedSearchTweetCandidate, + author: Option[User], + copyIds: CopyIds +)( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with SubscribedSearchTweetCandidate + with TweetAuthorDetails + with SubscribedSearchTweetIbis2Hydrator + with SubscribedSearchTweetNtabRequestHydrator { + override def tweetAuthor: Future[Option[User]] = Future.value(author) + + override def weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + + override def tweetId: Long = candidate.tweetId + + override def pushCopyId: Option[Int] = copyIds.pushCopyId + + override def ntabCopyId: Option[Int] = copyIds.ntabCopyId + + override def copyAggregationId: Option[String] = copyIds.aggregationId + + override def target: PushTypes.Target = candidate.target + + override def searchTerm: String = candidate.searchTerm + + override def timeBoundedLandingUrl: Option[String] = None + + override def statsReceiver: StatsReceiver = stats + + override def tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult +} + +case class SubscribedSearchTweetCandidatePredicates(override val config: Config) + extends BasicTweetPredicatesForRFPH[SubscribedSearchTweetPushCandidate] { + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.scala new file mode 100644 index 0000000000..b04a16ac33 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopTweetImpressionsPushCandidate.scala @@ -0,0 +1,70 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TopTweetImpressionsCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.TopTweetImpressionsCandidateIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.TopTweetImpressionsNTabRequestHydrator +import com.twitter.frigate.pushservice.predicate.TopTweetImpressionsPredicates +import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.notificationservice.thriftscala.StoryContext +import com.twitter.notificationservice.thriftscala.StoryContextValue +import com.twitter.stitch.tweetypie.TweetyPie + +/** + * This class defines a hydrated [[TopTweetImpressionsCandidate]] + * + * @param candidate: [[TopTweetImpressionsCandidate]] for the candidate representing the user's Tweet with the most impressions + * @param copyIds: push and ntab notification copy + * @param stats: finagle scoped states receiver + * @param pushModelScorer: ML model score object for fetching prediction scores + */ +class TopTweetImpressionsPushCandidate( + candidate: RawCandidate with TopTweetImpressionsCandidate, + copyIds: CopyIds +)( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with TopTweetImpressionsCandidate + with TopTweetImpressionsNTabRequestHydrator + with TopTweetImpressionsCandidateIbis2Hydrator { + override val target: PushTypes.Target = candidate.target + override val commonRecType: CommonRecommendationType = candidate.commonRecType + override val tweetId: Long = candidate.tweetId + override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = + candidate.tweetyPieResult + override val impressionsCount: Long = candidate.impressionsCount + + override val statsReceiver: StatsReceiver = stats.scope(getClass.getSimpleName) + override val pushCopyId: Option[Int] = copyIds.pushCopyId + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + override val copyAggregationId: Option[String] = copyIds.aggregationId + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + override val storyContext: Option[StoryContext] = + Some(StoryContext(altText = "", value = Some(StoryContextValue.Tweets(Seq(tweetId))))) +} + +case class TopTweetImpressionsPushCandidatePredicates(config: Config) + extends BasicTweetPredicatesForRFPH[TopTweetImpressionsPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val preCandidateSpecificPredicates: List[ + NamedPredicate[TopTweetImpressionsPushCandidate] + ] = List( + TopTweetImpressionsPredicates.topTweetImpressionsFatiguePredicate + ) + + override val postCandidateSpecificPredicates: List[ + NamedPredicate[TopTweetImpressionsPushCandidate] + ] = List( + TopTweetImpressionsPredicates.topTweetImpressionsThreshold() + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.scala new file mode 100644 index 0000000000..f89eb28bf9 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TopicProofTweetPushCandidate.scala @@ -0,0 +1,71 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TopicProofTweetCandidate +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.TopicProofTweetIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.TopicProofTweetNtabRequestHydrator +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.stitch.tweetypie.TweetyPie +import com.twitter.util.Future + +/** + * This class defines a hydrated [[TopicProofTweetCandidate]] + * + * @param candidate : [[TopicProofTweetCandidate]] for the candidate representint a Tweet recommendation for followed Topic + * @param author : Tweet author representated as Gizmoduck user object + * @param copyIds : push and ntab notification copy + * @param stats : finagle scoped states receiver + * @param pushModelScorer : ML model score object for fetching prediction scores + */ +class TopicProofTweetPushCandidate( + candidate: RawCandidate with TopicProofTweetCandidate, + author: Option[User], + copyIds: CopyIds +)( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with TopicProofTweetCandidate + with TweetAuthorDetails + with TopicProofTweetNtabRequestHydrator + with TopicProofTweetIbis2Hydrator { + override val statsReceiver: StatsReceiver = stats + override val target: PushTypes.Target = candidate.target + override val tweetId: Long = candidate.tweetId + override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + override val pushCopyId: Option[Int] = copyIds.pushCopyId + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + override val copyAggregationId: Option[String] = copyIds.aggregationId + override val semanticCoreEntityId = candidate.semanticCoreEntityId + override val localizedUttEntity = candidate.localizedUttEntity + override val tweetAuthor = Future.value(author) + override val topicListingSetting = candidate.topicListingSetting + override val algorithmCR = candidate.algorithmCR + override val commonRecType: CommonRecommendationType = candidate.commonRecType + override val tagsCR = candidate.tagsCR + override val isOutOfNetwork = candidate.isOutOfNetwork +} + +case class TopicProofTweetCandidatePredicates(override val config: Config) + extends BasicTweetPredicatesForRFPH[TopicProofTweetPushCandidate] { + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val preCandidateSpecificPredicates: List[NamedPredicate[TopicProofTweetPushCandidate]] = + List( + PredicatesForCandidate.paramPredicate( + PushFeatureSwitchParams.EnableTopicProofTweetRecs + ), + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.scala new file mode 100644 index 0000000000..ec580f629f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TrendTweetPushCandidate.scala @@ -0,0 +1,50 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.events.recos.thriftscala.TrendsContext +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TrendTweetCandidate +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.TrendTweetIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.TrendTweetNtabHydrator +import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.stitch.tweetypie.TweetyPie +import com.twitter.util.Future + +class TrendTweetPushCandidate( + candidate: RawCandidate with TrendTweetCandidate, + author: Option[User], + copyIds: CopyIds +)( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with TrendTweetCandidate + with TweetAuthorDetails + with TrendTweetIbis2Hydrator + with TrendTweetNtabHydrator { + override val statsReceiver: StatsReceiver = stats + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + override val tweetId: Long = candidate.tweetId + override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult + override lazy val tweetAuthor: Future[Option[User]] = Future.value(author) + override val target: PushTypes.Target = candidate.target + override val landingUrl: String = candidate.landingUrl + override val timeBoundedLandingUrl: Option[String] = candidate.timeBoundedLandingUrl + override val pushCopyId: Option[Int] = copyIds.pushCopyId + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + override val trendId: String = candidate.trendId + override val trendName: String = candidate.trendName + override val copyAggregationId: Option[String] = copyIds.aggregationId + override val context: TrendsContext = candidate.context +} + +case class TrendTweetPredicates(override val config: Config) + extends BasicTweetPredicatesForRFPH[TrendTweetPushCandidate] { + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.scala new file mode 100644 index 0000000000..1981e7bb55 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TripTweetPushCandidate.scala @@ -0,0 +1,60 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate +import com.twitter.frigate.common.base.TopicCandidate +import com.twitter.frigate.common.base.TripCandidate +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.OutOfNetworkTweetIbis2HydratorForCandidate +import com.twitter.frigate.pushservice.model.ntab.OutOfNetworkTweetNTabRequestHydrator +import com.twitter.frigate.pushservice.take.predicates.OutOfNetworkTweetPredicates +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.stitch.tweetypie.TweetyPie +import com.twitter.topiclisting.utt.LocalizedEntity +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import com.twitter.util.Future + +class TripTweetPushCandidate( + candidate: RawCandidate with OutOfNetworkTweetCandidate with TripCandidate, + author: Future[Option[User]], + copyIds: CopyIds +)( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with TripCandidate + with TopicCandidate + with OutOfNetworkTweetCandidate + with TweetAuthorDetails + with OutOfNetworkTweetNTabRequestHydrator + with OutOfNetworkTweetIbis2HydratorForCandidate { + override val statsReceiver: StatsReceiver = stats + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + override val tweetId: Long = candidate.tweetId + override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = + candidate.tweetyPieResult + override lazy val tweetAuthor: Future[Option[User]] = author + override val target: PushTypes.Target = candidate.target + override lazy val commonRecType: CommonRecommendationType = + candidate.commonRecType + override val pushCopyId: Option[Int] = copyIds.pushCopyId + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + override val copyAggregationId: Option[String] = copyIds.aggregationId + override lazy val semanticCoreEntityId: Option[Long] = None + override lazy val localizedUttEntity: Option[LocalizedEntity] = None + override lazy val algorithmCR: Option[String] = None + override val tripDomain: Option[collection.Set[TripDomain]] = candidate.tripDomain +} + +case class TripTweetCandidatePredicates(override val config: Config) + extends OutOfNetworkTweetPredicates[TripTweetPushCandidate] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.scala new file mode 100644 index 0000000000..72453224d8 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetAction.scala @@ -0,0 +1,26 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.SocialContextActions +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.base.TweetDetails +import com.twitter.frigate.pushservice.model.PushTypes._ +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.predicate._ +import com.twitter.frigate.pushservice.take.predicates.BasicTweetPredicatesForRFPH + +case class TweetActionCandidatePredicates(override val config: Config) + extends BasicTweetPredicatesForRFPH[ + PushCandidate with TweetCandidate with TweetDetails with SocialContextActions + ] { + + implicit val statsReceiver: StatsReceiver = config.statsReceiver.scope(getClass.getSimpleName) + + override val preCandidateSpecificPredicates = List(PredicatesForCandidate.minSocialContext(1)) + + override val postCandidateSpecificPredicates = List( + PredicatesForCandidate.socialContextBeingFollowed(config.edgeStore), + PredicatesForCandidate.socialContextBlockingOrMuting(config.edgeStore), + PredicatesForCandidate.socialContextNotRetweetFollowing(config.edgeStore) + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.scala new file mode 100644 index 0000000000..ae31ddc6c2 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetFavorite.scala @@ -0,0 +1,53 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.SocialContextAction +import com.twitter.frigate.common.base.SocialContextUserDetails +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetFavoriteCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.TweetFavoriteCandidateIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.TweetFavoriteNTabRequestHydrator +import com.twitter.frigate.pushservice.util.CandidateHydrationUtil.TweetWithSocialContextTraits +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.stitch.tweetypie.TweetyPie +import com.twitter.util.Future + +class TweetFavoritePushCandidate( + candidate: RawCandidate with TweetWithSocialContextTraits, + socialContextUserMap: Future[Map[Long, Option[User]]], + author: Future[Option[User]], + copyIds: CopyIds +)( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with TweetFavoriteCandidate + with SocialContextUserDetails + with TweetAuthorDetails + with TweetFavoriteNTabRequestHydrator + with TweetFavoriteCandidateIbis2Hydrator { + override val statsReceiver: StatsReceiver = stats + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + override val tweetId: Long = candidate.tweetId + override val socialContextActions: Seq[SocialContextAction] = + candidate.socialContextActions + + override val socialContextAllTypeActions: Seq[SocialContextAction] = + candidate.socialContextAllTypeActions + + override lazy val scUserMap: Future[Map[Long, Option[User]]] = socialContextUserMap + override lazy val tweetAuthor: Future[Option[User]] = author + override lazy val commonRecType: CommonRecommendationType = + candidate.commonRecType + override val target: PushTypes.Target = candidate.target + override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = + candidate.tweetyPieResult + override val pushCopyId: Option[Int] = copyIds.pushCopyId + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + override val copyAggregationId: Option[String] = copyIds.aggregationId +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.scala new file mode 100644 index 0000000000..61c8c6526d --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/TweetRetweet.scala @@ -0,0 +1,51 @@ +package com.twitter.frigate.pushservice.model + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.SocialContextAction +import com.twitter.frigate.common.base.SocialContextUserDetails +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetRetweetCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.model.ibis.TweetRetweetCandidateIbis2Hydrator +import com.twitter.frigate.pushservice.model.ntab.TweetRetweetNTabRequestHydrator +import com.twitter.frigate.pushservice.util.CandidateHydrationUtil.TweetWithSocialContextTraits +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.stitch.tweetypie.TweetyPie +import com.twitter.util.Future + +class TweetRetweetPushCandidate( + candidate: RawCandidate with TweetWithSocialContextTraits, + socialContextUserMap: Future[Map[Long, Option[User]]], + author: Future[Option[User]], + copyIds: CopyIds +)( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer) + extends PushCandidate + with TweetRetweetCandidate + with SocialContextUserDetails + with TweetAuthorDetails + with TweetRetweetNTabRequestHydrator + with TweetRetweetCandidateIbis2Hydrator { + override val statsReceiver: StatsReceiver = stats + override val weightedOpenOrNtabClickModelScorer: PushMLModelScorer = pushModelScorer + override val tweetId: Long = candidate.tweetId + override val socialContextActions: Seq[SocialContextAction] = + candidate.socialContextActions + + override val socialContextAllTypeActions: Seq[SocialContextAction] = + candidate.socialContextAllTypeActions + + override lazy val scUserMap: Future[Map[Long, Option[User]]] = socialContextUserMap + override lazy val tweetAuthor: Future[Option[User]] = author + override lazy val commonRecType: CommonRecommendationType = candidate.commonRecType + override val target: PushTypes.Target = candidate.target + override lazy val tweetyPieResult: Option[TweetyPie.TweetyPieResult] = candidate.tweetyPieResult + override val pushCopyId: Option[Int] = copyIds.pushCopyId + override val ntabCopyId: Option[Int] = copyIds.ntabCopyId + override val copyAggregationId: Option[String] = copyIds.aggregationId +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.scala new file mode 100644 index 0000000000..11cc0617ab --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/CopyInfo.scala @@ -0,0 +1,33 @@ +package com.twitter.frigate.pushservice.model.candidate + +import com.twitter.frigate.common.util.MRPushCopy +import com.twitter.frigate.common.util.MrPushCopyObjects +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.util.CandidateUtil + +case class CopyIds( + pushCopyId: Option[Int] = None, + ntabCopyId: Option[Int] = None, + aggregationId: Option[String] = None) + +trait CopyInfo { + self: PushCandidate => + + import com.twitter.frigate.data_pipeline.common.FrigateNotificationUtil._ + + def getPushCopy: Option[MRPushCopy] = + pushCopyId match { + case Some(pushCopyId) => MrPushCopyObjects.getCopyFromId(pushCopyId) + case _ => + crt2PushCopy( + commonRecType, + CandidateUtil.getSocialContextActionsFromCandidate(self).size + ) + } + + def pushCopyId: Option[Int] + + def ntabCopyId: Option[Int] + + def copyAggregationId: Option[String] +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.scala new file mode 100644 index 0000000000..4ba79f4852 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/MLScores.scala @@ -0,0 +1,307 @@ +package com.twitter.frigate.pushservice.model.candidate + +import com.twitter.frigate.common.base.FeatureMap +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.ml.HydrationContextBuilder +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushMLModel +import com.twitter.frigate.pushservice.params.WeightedOpenOrNtabClickModel +import com.twitter.nrel.hydration.push.HydrationContext +import com.twitter.timelines.configapi.FSParam +import com.twitter.util.Future +import java.util.concurrent.ConcurrentHashMap +import scala.collection.concurrent.{Map => CMap} +import scala.collection.convert.decorateAsScala._ + +trait MLScores { + + self: PushCandidate => + + lazy val candidateHydrationContext: Future[HydrationContext] = HydrationContextBuilder.build(self) + + def weightedOpenOrNtabClickModelScorer: PushMLModelScorer + + // Used to store the scores and avoid duplicate prediction + private val qualityModelScores: CMap[ + (PushMLModel.Value, WeightedOpenOrNtabClickModel.ModelNameType), + Future[Option[Double]] + ] = + new ConcurrentHashMap[(PushMLModel.Value, WeightedOpenOrNtabClickModel.ModelNameType), Future[ + Option[Double] + ]]().asScala + + def populateQualityModelScore( + pushMLModel: PushMLModel.Value, + modelVersion: WeightedOpenOrNtabClickModel.ModelNameType, + prob: Future[Option[Double]] + ) = { + val modelAndVersion = (pushMLModel, modelVersion) + if (!qualityModelScores.contains(modelAndVersion)) { + qualityModelScores += modelAndVersion -> prob + } + } + + // The ML scores that also depend on other candidates and are only available after all candidates are processed + // For example, the likelihood info for Importance Sampling + private lazy val crossCandidateMlScores: CMap[String, Double] = + new ConcurrentHashMap[String, Double]().asScala + + def populateCrossCandidateMlScores(scoreName: String, score: Double): Unit = { + if (crossCandidateMlScores.contains(scoreName)) { + throw new Exception( + s"$scoreName has been populated in the CrossCandidateMlScores!\n" + + s"Existing crossCandidateMlScores are ${crossCandidateMlScores}\n" + ) + } + crossCandidateMlScores += scoreName -> score + } + + def getMLModelScore( + pushMLModel: PushMLModel.Value, + modelVersion: WeightedOpenOrNtabClickModel.ModelNameType + ): Future[Option[Double]] = { + qualityModelScores.getOrElseUpdate( + (pushMLModel, modelVersion), + weightedOpenOrNtabClickModelScorer + .singlePredicationForModelVersion(modelVersion, self, Some(pushMLModel)) + ) + } + + def getMLModelScoreWithoutUpdate( + pushMLModel: PushMLModel.Value, + modelVersion: WeightedOpenOrNtabClickModel.ModelNameType + ): Future[Option[Double]] = { + qualityModelScores.getOrElse( + (pushMLModel, modelVersion), + Future.None + ) + } + + def getWeightedOpenOrNtabClickModelScore( + weightedOONCModelParam: FSParam[WeightedOpenOrNtabClickModel.ModelNameType] + ): Future[Option[Double]] = { + getMLModelScore( + PushMLModel.WeightedOpenOrNtabClickProbability, + target.params(weightedOONCModelParam) + ) + } + + /* After we unify the ranking and filtering models, we follow the iteration process below + When improving the WeightedOONC model, + 1) Run experiment which only replace the ranking model + 2) Make decisions according to the experiment results + 3) Use the ranking model for filtering + 4) Adjust percentile thresholds if necessary + */ + lazy val mrWeightedOpenOrNtabClickRankingProbability: Future[Option[Double]] = + target.rankingModelParam.flatMap { modelParam => + getWeightedOpenOrNtabClickModelScore(modelParam) + } + + def getBigFilteringScore( + pushMLModel: PushMLModel.Value, + modelVersion: WeightedOpenOrNtabClickModel.ModelNameType + ): Future[Option[Double]] = { + mrWeightedOpenOrNtabClickRankingProbability.flatMap { + case Some(rankingScore) => + // Adds ranking score to feature map (we must ensure the feature key is also in the feature context) + mergeFeatures( + FeatureMap( + numericFeatures = Map("scribe.WeightedOpenOrNtabClickProbability" -> rankingScore) + ) + ) + getMLModelScore(pushMLModel, modelVersion) + case _ => Future.None + } + } + + def getWeightedOpenOrNtabClickScoreForScribing(): Seq[Future[Map[String, Double]]] = { + Seq( + mrWeightedOpenOrNtabClickRankingProbability.map { + case Some(score) => Map(PushMLModel.WeightedOpenOrNtabClickProbability.toString -> score) + case _ => Map.empty[String, Double] + }, + Future + .join( + target.rankingModelParam, + mrWeightedOpenOrNtabClickRankingProbability + ).map { + case (rankingModelParam, Some(score)) => + Map(target.params(rankingModelParam).toString -> score) + case _ => Map.empty[String, Double] + } + ) + } + + def getNsfwScoreForScribing(): Seq[Future[Map[String, Double]]] = { + val nsfwScoreFut = getMLModelScoreWithoutUpdate( + PushMLModel.HealthNsfwProbability, + target.params(PushFeatureSwitchParams.BqmlHealthModelTypeParam)) + Seq(nsfwScoreFut.map { nsfwScoreOpt => + nsfwScoreOpt + .map(nsfwScore => Map(PushMLModel.HealthNsfwProbability.toString -> nsfwScore)).getOrElse( + Map.empty[String, Double]) + }) + } + + def getBigFilteringSupervisedScoresForScribing(): Seq[Future[Map[String, Double]]] = { + if (target.params( + PushFeatureSwitchParams.EnableMrRequestScribingBigFilteringSupervisedScores)) { + Seq( + mrBigFilteringSupervisedSendingScore.map { + case Some(score) => + Map(PushMLModel.BigFilteringSupervisedSendingModel.toString -> score) + case _ => Map.empty[String, Double] + }, + mrBigFilteringSupervisedWithoutSendingScore.map { + case Some(score) => + Map(PushMLModel.BigFilteringSupervisedWithoutSendingModel.toString -> score) + case _ => Map.empty[String, Double] + } + ) + } else Seq.empty[Future[Map[String, Double]]] + } + + def getBigFilteringRLScoresForScribing(): Seq[Future[Map[String, Double]]] = { + if (target.params(PushFeatureSwitchParams.EnableMrRequestScribingBigFilteringRLScores)) { + Seq( + mrBigFilteringRLSendingScore.map { + case Some(score) => Map(PushMLModel.BigFilteringRLSendingModel.toString -> score) + case _ => Map.empty[String, Double] + }, + mrBigFilteringRLWithoutSendingScore.map { + case Some(score) => Map(PushMLModel.BigFilteringRLWithoutSendingModel.toString -> score) + case _ => Map.empty[String, Double] + } + ) + } else Seq.empty[Future[Map[String, Double]]] + } + + def buildModelScoresSeqForScribing(): Seq[Future[Map[String, Double]]] = { + getWeightedOpenOrNtabClickScoreForScribing() ++ + getBigFilteringSupervisedScoresForScribing() ++ + getBigFilteringRLScoresForScribing() ++ + getNsfwScoreForScribing() + } + + lazy val mrBigFilteringSupervisedSendingScore: Future[Option[Double]] = + getBigFilteringScore( + PushMLModel.BigFilteringSupervisedSendingModel, + target.params(PushFeatureSwitchParams.BigFilteringSupervisedSendingModelParam) + ) + + lazy val mrBigFilteringSupervisedWithoutSendingScore: Future[Option[Double]] = + getBigFilteringScore( + PushMLModel.BigFilteringSupervisedWithoutSendingModel, + target.params(PushFeatureSwitchParams.BigFilteringSupervisedWithoutSendingModelParam) + ) + + lazy val mrBigFilteringRLSendingScore: Future[Option[Double]] = + getBigFilteringScore( + PushMLModel.BigFilteringRLSendingModel, + target.params(PushFeatureSwitchParams.BigFilteringRLSendingModelParam) + ) + + lazy val mrBigFilteringRLWithoutSendingScore: Future[Option[Double]] = + getBigFilteringScore( + PushMLModel.BigFilteringRLWithoutSendingModel, + target.params(PushFeatureSwitchParams.BigFilteringRLWithoutSendingModelParam) + ) + + lazy val mrWeightedOpenOrNtabClickFilteringProbability: Future[Option[Double]] = + getWeightedOpenOrNtabClickModelScore( + target.filteringModelParam + ) + + lazy val mrQualityUprankingProbability: Future[Option[Double]] = + getMLModelScore( + PushMLModel.FilteringProbability, + target.params(PushFeatureSwitchParams.QualityUprankingModelTypeParam) + ) + + lazy val mrNsfwScore: Future[Option[Double]] = + getMLModelScoreWithoutUpdate( + PushMLModel.HealthNsfwProbability, + target.params(PushFeatureSwitchParams.BqmlHealthModelTypeParam) + ) + + // MR quality upranking param + private val qualityUprankingBoost: String = "QualityUprankingBoost" + private val producerQualityUprankingBoost: String = "ProducerQualityUprankingBoost" + private val qualityUprankingInfo: CMap[String, Double] = + new ConcurrentHashMap[String, Double]().asScala + + lazy val mrQualityUprankingBoost: Option[Double] = + qualityUprankingInfo.get(qualityUprankingBoost) + lazy val mrProducerQualityUprankingBoost: Option[Double] = + qualityUprankingInfo.get(producerQualityUprankingBoost) + + def setQualityUprankingBoost(boost: Double) = + if (qualityUprankingInfo.contains(qualityUprankingBoost)) { + qualityUprankingInfo(qualityUprankingBoost) = boost + } else { + qualityUprankingInfo += qualityUprankingBoost -> boost + } + def setProducerQualityUprankingBoost(boost: Double) = + if (qualityUprankingInfo.contains(producerQualityUprankingBoost)) { + qualityUprankingInfo(producerQualityUprankingBoost) = boost + } else { + qualityUprankingInfo += producerQualityUprankingBoost -> boost + } + + private lazy val mrModelScoresFut: Future[Map[String, Double]] = { + if (self.target.isLoggedOutUser) { + Future.value(Map.empty[String, Double]) + } else { + Future + .collectToTry { + buildModelScoresSeqForScribing() + }.map { scoreTrySeq => + scoreTrySeq + .collect { + case result if result.isReturn => result.get() + }.reduce(_ ++ _) + } + } + } + + // Internal model scores (scores that are independent of other candidates) for scribing + lazy val modelScores: Future[Map[String, Double]] = + target.dauProbability.flatMap { dauProbabilityOpt => + val dauProbScoreMap = dauProbabilityOpt + .map(_.probability).map { dauProb => + PushMLModel.DauProbability.toString -> dauProb + }.toMap + + // Avoid unnecessary MR model scribing + if (target.isDarkWrite) { + mrModelScoresFut.map(dauProbScoreMap ++ _) + } else if (RecTypes.isSendHandlerType(commonRecType) && !RecTypes + .sendHandlerTypesUsingMrModel(commonRecType)) { + Future.value(dauProbScoreMap) + } else { + mrModelScoresFut.map(dauProbScoreMap ++ _) + } + } + + // We will scribe both internal ML scores and cross-Candidate scores + def getModelScoresforScribing(): Future[Map[String, Double]] = { + if (RecTypes.notEligibleForModelScoreTracking(commonRecType) || self.target.isLoggedOutUser) { + Future.value(Map.empty[String, Double]) + } else { + modelScores.map { internalScores => + if (internalScores.keySet.intersect(crossCandidateMlScores.keySet).nonEmpty) { + throw new Exception( + "crossCandidateMlScores overlap internalModelScores\n" + + s"internalScores keySet: ${internalScores.keySet}\n" + + s"crossCandidateScores keySet: ${crossCandidateMlScores.keySet}\n" + ) + } + + internalScores ++ crossCandidateMlScores + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.scala new file mode 100644 index 0000000000..283c3d97c5 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/QualityScribing.scala @@ -0,0 +1,104 @@ +package com.twitter.frigate.pushservice.model.candidate + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.HighQualityScribingScores +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushMLModel +import com.twitter.util.Future +import java.util.concurrent.ConcurrentHashMap +import scala.collection.concurrent.{Map => CMap} +import scala.collection.convert.decorateAsScala._ + +trait QualityScribing { + self: PushCandidate with MLScores => + + // Use to store other scores (to avoid duplicate queries to other services, e.g. HSS) + private val externalCachedScores: CMap[String, Future[Option[Double]]] = + new ConcurrentHashMap[String, Future[Option[Double]]]().asScala + + /** + * Retrieves the model version as specified by the corresponding FS param. + * This model version will be used for getting the cached score or triggering + * a prediction request. + * + * @param modelName The score we will like to scribe + */ + private def getModelVersion( + modelName: HighQualityScribingScores.Name + ): String = { + modelName match { + case HighQualityScribingScores.HeavyRankingScore => + target.params(PushFeatureSwitchParams.HighQualityCandidatesHeavyRankingModel) + case HighQualityScribingScores.NonPersonalizedQualityScoreUsingCnn => + target.params(PushFeatureSwitchParams.HighQualityCandidatesNonPersonalizedQualityCnnModel) + case HighQualityScribingScores.BqmlNsfwScore => + target.params(PushFeatureSwitchParams.HighQualityCandidatesBqmlNsfwModel) + case HighQualityScribingScores.BqmlReportScore => + target.params(PushFeatureSwitchParams.HighQualityCandidatesBqmlReportModel) + } + } + + /** + * Retrieves the score for scribing either from a cached value or + * by generating a prediction request. This will increase model QPS + * + * @param pushMLModel This represents the prefix of the model name (i.e. [pushMLModel]_[version]) + * @param scoreName The name to be use when scribing this score + */ + def getScribingScore( + pushMLModel: PushMLModel.Value, + scoreName: HighQualityScribingScores.Name + ): Future[(String, Option[Double])] = { + getMLModelScore( + pushMLModel, + getModelVersion(scoreName) + ).map { scoreOpt => + scoreName.toString -> scoreOpt + } + } + + /** + * Retrieves the score for scribing if it has been computed/cached before otherwise + * it will return Future.None + * + * @param pushMLModel This represents the prefix of the model name (i.e. [pushMLModel]_[version]) + * @param scoreName The name to be use when scribing this score + */ + def getScribingScoreWithoutUpdate( + pushMLModel: PushMLModel.Value, + scoreName: HighQualityScribingScores.Name + ): Future[(String, Option[Double])] = { + getMLModelScoreWithoutUpdate( + pushMLModel, + getModelVersion(scoreName) + ).map { scoreOpt => + scoreName.toString -> scoreOpt + } + } + + /** + * Caches the given score future + * + * @param scoreName The name to be use when scribing this score + * @param scoreFut Future mapping scoreName -> scoreOpt + */ + def cacheExternalScore(scoreName: String, scoreFut: Future[Option[Double]]) = { + if (!externalCachedScores.contains(scoreName)) { + externalCachedScores += scoreName -> scoreFut + } + } + + /** + * Returns all external scores future cached as a sequence + */ + def getExternalCachedScores: Seq[Future[(String, Option[Double])]] = { + externalCachedScores.map { + case (modelName, scoreFut) => + scoreFut.map { scoreOpt => modelName -> scoreOpt } + }.toSeq + } + + def getExternalCachedScoreByName(name: String): Future[Option[Double]] = { + externalCachedScores.getOrElse(name, Future.None) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.scala new file mode 100644 index 0000000000..a43530b447 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/candidate/Scriber.scala @@ -0,0 +1,277 @@ +package com.twitter.frigate.pushservice.model.candidate + +import com.twitter.frigate.data_pipeline.features_common.PushQualityModelFeatureContext.featureContext +import com.twitter.frigate.data_pipeline.features_common.PushQualityModelUtil +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.common.util.NotificationScribeUtil +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.OutOfNetworkTweetPushCandidate +import com.twitter.frigate.pushservice.model.TopicProofTweetPushCandidate +import com.twitter.frigate.pushservice.ml.HydrationContextBuilder +import com.twitter.frigate.pushservice.predicate.quality_model_predicate.PDauCohort +import com.twitter.frigate.pushservice.predicate.quality_model_predicate.PDauCohortUtil +import com.twitter.frigate.pushservice.util.Candidate2FrigateNotification +import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil.sensitiveMediaCategoryFeatureName +import com.twitter.frigate.scribe.thriftscala.FrigateNotificationScribeType +import com.twitter.frigate.scribe.thriftscala.NotificationScribe +import com.twitter.frigate.scribe.thriftscala.PredicateDetailedInfo +import com.twitter.frigate.scribe.thriftscala.PushCapInfo +import com.twitter.frigate.thriftscala.ChannelName +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.frigate.thriftscala.OverrideInfo +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.model.user_state.UserState.UserState +import com.twitter.ibis2.service.thriftscala.Ibis2Response +import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions +import com.twitter.nrel.heavyranker.FeatureHydrator +import com.twitter.util.Future +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import scala.collection.concurrent.{Map => CMap} +import scala.collection.Map +import scala.collection.convert.decorateAsScala._ + +trait Scriber { + self: PushCandidate => + + def statsReceiver: StatsReceiver + + def frigateNotification: FrigateNotification = Candidate2FrigateNotification + .getFrigateNotification(self)(statsReceiver) + .copy(copyAggregationId = self.copyAggregationId) + + lazy val impressionId: String = UUID.randomUUID.toString.replaceAll("-", "") + + // Used to store the score and threshold for predicates + // Map(predicate name, (score, threshold, filter?)) + private val predicateScoreAndThreshold: CMap[String, PredicateDetailedInfo] = + new ConcurrentHashMap[String, PredicateDetailedInfo]().asScala + + def cachePredicateInfo( + predName: String, + predScore: Double, + predThreshold: Double, + predResult: Boolean, + additionalInformation: Option[Map[String, Double]] = None + ) = { + if (!predicateScoreAndThreshold.contains(predName)) { + predicateScoreAndThreshold += predName -> PredicateDetailedInfo( + predName, + predScore, + predThreshold, + predResult, + additionalInformation) + } + } + + def getCachedPredicateInfo(): Seq[PredicateDetailedInfo] = predicateScoreAndThreshold.values.toSeq + + def frigateNotificationForPersistence( + channels: Seq[ChannelName], + isSilentPush: Boolean, + overrideInfoOpt: Option[OverrideInfo] = None, + copyFeaturesList: Set[String] + ): Future[FrigateNotification] = { + + // record display location for frigate notification + statsReceiver + .scope("FrigateNotificationForPersistence") + .scope("displayLocation") + .counter(frigateNotification.notificationDisplayLocation.name) + .incr() + + val getModelScores = self.getModelScoresforScribing() + + Future.join(getModelScores, self.target.targetMrUserState).map { + case (mlScores, mrUserState) => + frigateNotification.copy( + impressionId = Some(impressionId), + isSilentPush = Some(isSilentPush), + overrideInfo = overrideInfoOpt, + mlModelScores = Some(mlScores), + mrUserState = mrUserState.map(_.name), + copyFeatures = Some(copyFeaturesList.toSeq) + ) + } + } + // scribe data + private def getNotificationScribe( + notifForPersistence: FrigateNotification, + userState: Option[UserState], + dauCohort: PDauCohort.Value, + ibis2Response: Option[Ibis2Response], + tweetAuthorId: Option[Long], + recUserId: Option[Long], + modelScoresMap: Option[Map[String, Double]], + primaryClient: Option[String], + isMrBackfillCR: Option[Boolean] = None, + tagsCR: Option[Seq[String]] = None, + gizmoduckTargetUser: Option[User], + predicateDetailedInfoList: Option[Seq[PredicateDetailedInfo]] = None, + pushCapInfoList: Option[Seq[PushCapInfo]] = None + ): NotificationScribe = { + NotificationScribe( + FrigateNotificationScribeType.SendMessage, + System.currentTimeMillis(), + targetUserId = Some(self.target.targetId), + timestampKeyForHistoryV2 = Some(createdAt.inSeconds), + sendType = NotificationScribeUtil.convertToScribeDisplayLocation( + self.frigateNotification.notificationDisplayLocation + ), + recommendationType = NotificationScribeUtil.convertToScribeRecommendationType( + self.frigateNotification.commonRecommendationType + ), + commonRecommendationType = Some(self.frigateNotification.commonRecommendationType), + fromPushService = Some(true), + frigateNotification = Some(notifForPersistence), + impressionId = Some(impressionId), + skipModelInfo = target.skipModelInfo, + ibis2Response = ibis2Response, + tweetAuthorId = tweetAuthorId, + scribeFeatures = Some(target.noSkipButScribeFeatures), + userState = userState.map(_.toString), + pDauCohort = Some(dauCohort.toString), + recommendedUserId = recUserId, + modelScores = modelScoresMap, + primaryClient = primaryClient, + isMrBackfillCR = isMrBackfillCR, + tagsCR = tagsCR, + targetUserType = gizmoduckTargetUser.map(_.userType), + predicateDetailedInfoList = predicateDetailedInfoList, + pushCapInfoList = pushCapInfoList + ) + } + + def scribeData( + ibis2Response: Option[Ibis2Response] = None, + isSilentPush: Boolean = false, + overrideInfoOpt: Option[OverrideInfo] = None, + copyFeaturesList: Set[String] = Set.empty, + channels: Seq[ChannelName] = Seq.empty + ): Future[NotificationScribe] = { + + val recTweetAuthorId = self match { + case t: TweetCandidate with TweetAuthor => t.authorId + case _ => None + } + + val recUserId = self match { + case u: UserCandidate => Some(u.userId) + case _ => None + } + + val isMrBackfillCR = self match { + case t: OutOfNetworkTweetPushCandidate => t.isMrBackfillCR + case _ => None + } + + val tagsCR = self match { + case t: OutOfNetworkTweetPushCandidate => + t.tagsCR.map { tags => + tags.map(_.toString) + } + case t: TopicProofTweetPushCandidate => + t.tagsCR.map { tags => + tags.map(_.toString) + } + case _ => None + } + + Future + .join( + frigateNotificationForPersistence( + channels = channels, + isSilentPush = isSilentPush, + overrideInfoOpt = overrideInfoOpt, + copyFeaturesList = copyFeaturesList + ), + target.targetUserState, + PDauCohortUtil.getPDauCohort(target), + target.deviceInfo, + target.targetUser + ) + .flatMap { + case (notifForPersistence, userState, dauCohort, deviceInfo, gizmoduckTargetUserOpt) => + val primaryClient = deviceInfo.flatMap(_.guessedPrimaryClient).map(_.toString) + val cachedPredicateInfo = + if (self.target.params(PushParams.EnablePredicateDetailedInfoScribing)) { + Some(getCachedPredicateInfo()) + } else None + + val cachedPushCapInfo = + if (self.target + .params(PushParams.EnablePushCapInfoScribing)) { + Some(target.finalPushcapAndFatigue.values.toSeq) + } else None + + val data = getNotificationScribe( + notifForPersistence, + userState, + dauCohort, + ibis2Response, + recTweetAuthorId, + recUserId, + notifForPersistence.mlModelScores, + primaryClient, + isMrBackfillCR, + tagsCR, + gizmoduckTargetUserOpt, + cachedPredicateInfo, + cachedPushCapInfo + ) + //Don't scribe features for CRTs not eligible for ML Layer + if ((target.isModelTrainingData || target.scribeFeatureWithoutHydratingNewFeatures) + && !RecTypes.notEligibleForModelScoreTracking(self.commonRecType)) { + // scribe all the features for the model training data + self.getFeaturesForScribing.map { scribedFeatureMap => + if (target.params(PushParams.EnableScribingMLFeaturesAsDataRecord) && !target.params( + PushFeatureSwitchParams.EnableMrScribingMLFeaturesAsFeatureMapForStaging)) { + val scribedFeatureDataRecord = + ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord( + PushQualityModelUtil.adaptToDataRecord(scribedFeatureMap, featureContext)) + data.copy( + featureDataRecord = Some(scribedFeatureDataRecord) + ) + } else { + data.copy(features = + Some(PushQualityModelUtil.convertFeatureMapToFeatures(scribedFeatureMap))) + } + } + } else Future.value(data) + } + } + + def getFeaturesForScribing: Future[FeatureMap] = { + target.featureMap + .flatMap { targetFeatureMap => + val onlineFeatureMap = targetFeatureMap ++ self + .candidateFeatureMap() // targetFeatureMap includes target core user history features + + val filteredFeatureMap = { + onlineFeatureMap.copy( + sparseContinuousFeatures = onlineFeatureMap.sparseContinuousFeatures.filterKeys( + !_.equals(sensitiveMediaCategoryFeatureName)) + ) + } + + val targetHydrationContext = HydrationContextBuilder.build(self.target) + val candidateHydrationContext = HydrationContextBuilder.build(self) + + val featureMapFut = targetHydrationContext.join(candidateHydrationContext).flatMap { + case (targetContext, candidateContext) => + FeatureHydrator.getFeatures( + candidateHydrationContext = candidateContext, + targetHydrationContext = targetContext, + onlineFeatures = filteredFeatureMap, + statsReceiver = statsReceiver) + } + + featureMapFut + } + } + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.scala new file mode 100644 index 0000000000..75b00e3460 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/CustomConfigurationMapForIbis.scala @@ -0,0 +1,25 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.ibis2.lib.util.JsonMarshal +import com.twitter.util.Future + +trait CustomConfigurationMapForIbis { + self: PushCandidate => + + lazy val customConfigMapsJsonFut: Future[String] = { + customFieldsMapFut.map { customFields => + JsonMarshal.toJson(customFields) + } + } + + lazy val customConfigMapsFut: Future[Map[String, String]] = { + if (self.target.isLoggedOutUser) { + Future.value(Map.empty[String, String]) + } else { + customConfigMapsJsonFut.map { customConfigMapsJson => + Map("custom_config" -> customConfigMapsJson) + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.scala new file mode 100644 index 0000000000..a3a48ff288 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/DiscoverTwitterPushIbis2Hydrator.scala @@ -0,0 +1,17 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.common.base.DiscoverTwitterCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeFutModelValues +import com.twitter.util.Future + +trait DiscoverTwitterPushIbis2Hydrator extends Ibis2HydratorForCandidate { + self: PushCandidate with DiscoverTwitterCandidate => + + private lazy val targetModelValues: Map[String, String] = Map( + "target_user" -> target.targetId.toString + ) + + override lazy val modelValues: Future[Map[String, String]] = + mergeFutModelValues(super.modelValues, Future.value(targetModelValues)) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.scala new file mode 100644 index 0000000000..6ddaa49d1d --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/F1FirstDegreeTweetIbis2Hydrator.scala @@ -0,0 +1,24 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.F1FirstDegree +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.util.Future + +trait F1FirstDegreeTweetIbis2HydratorForCandidate + extends TweetCandidateIbis2Hydrator + with RankedSocialContextIbis2Hydrator { + self: PushCandidate with F1FirstDegree with TweetAuthorDetails => + + override lazy val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + + override lazy val tweetModelValues: Future[Map[String, String]] = { + for { + superModelValues <- super.tweetModelValues + tweetInlineModelValues <- tweetInlineActionModelValue + } yield { + superModelValues ++ otherModelValues ++ mediaModelValue ++ tweetInlineModelValues ++ inlineVideoMediaMap + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.scala new file mode 100644 index 0000000000..fd7d261863 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/Ibis2Hydrator.scala @@ -0,0 +1,127 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.common.util.MRPushCopy +import com.twitter.frigate.common.util.MrPushCopyObjects +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} +import com.twitter.ibis2.service.thriftscala.Flags +import com.twitter.ibis2.service.thriftscala.Ibis2Request +import com.twitter.ibis2.service.thriftscala.RecipientSelector +import com.twitter.ibis2.service.thriftscala.ResponseFlags +import com.twitter.util.Future +import scala.util.control.NoStackTrace +import com.twitter.ni.lib.logged_out_transform.Ibis2RequestTransform + +class PushCopyIdNotFoundException(private val message: String) + extends Exception(message) + with NoStackTrace + +class InvalidPushCopyIdException(private val message: String) + extends Exception(message) + with NoStackTrace + +trait Ibis2HydratorForCandidate + extends CandidatePushCopy + with OverrideForIbis2Request + with CustomConfigurationMapForIbis { + self: PushCandidate => + + lazy val silentPushModelValue: Map[String, String] = + if (RecTypes.silentPushDefaultEnabledCrts.contains(commonRecType)) { + Map.empty + } else { + Map("is_silent_push" -> "true") + } + + private def transformRelevanceScore( + mlScore: Double, + scoreRange: Seq[Double] + ): Double = { + val (lowerBound, upperBound) = (scoreRange.head, scoreRange.last) + (mlScore * (upperBound - lowerBound)) + lowerBound + } + + private def getBoundedMlScore(mlScore: Double): Double = { + if (RecTypes.isMagicFanoutEventType(commonRecType)) { + val mfScoreRange = target.params(FS.MagicFanoutRelevanceScoreRange) + transformRelevanceScore(mlScore, mfScoreRange) + } else { + val mrScoreRange = target.params(FS.MagicRecsRelevanceScoreRange) + transformRelevanceScore(mlScore, mrScoreRange) + } + } + + lazy val relevanceScoreMapFut: Future[Map[String, String]] = { + mrWeightedOpenOrNtabClickRankingProbability.map { + case Some(mlScore) if target.params(FS.IncludeRelevanceScoreInIbis2Payload) => + val boundedMlScore = getBoundedMlScore(mlScore) + Map("relevance_score" -> boundedMlScore.toString) + case _ => Map.empty[String, String] + } + } + + def customFieldsMapFut: Future[Map[String, String]] = relevanceScoreMapFut + + //override is only enabled for RFPH CRT + def modelValues: Future[Map[String, String]] = { + Future.join(overrideModelValueFut, customConfigMapsFut).map { + case (overrideModelValue, customConfig) => + overrideModelValue ++ silentPushModelValue ++ customConfig + } + } + + def modelName: String = pushCopy.ibisPushModelName + + def senderId: Option[Long] = None + + def ibis2Request: Future[Option[Ibis2Request]] = { + Future.join(self.target.loggedOutMetadata, modelValues).map { + case (Some(metadata), modelVals) => + Some( + Ibis2RequestTransform + .apply(metadata, modelName, modelVals).copy( + senderId = senderId, + flags = Some(Flags( + darkWrite = Some(target.isDarkWrite), + skipDupcheck = target.pushContext.flatMap(_.useDebugHandler), + responseFlags = Some(ResponseFlags(stringTelemetry = Some(true))) + )) + )) + case (None, modelVals) => + Some( + Ibis2Request( + recipientSelector = RecipientSelector(Some(target.targetId)), + modelName = modelName, + modelValues = Some(modelVals), + senderId = senderId, + flags = Some( + Flags( + darkWrite = Some(target.isDarkWrite), + skipDupcheck = target.pushContext.flatMap(_.useDebugHandler), + responseFlags = Some(ResponseFlags(stringTelemetry = Some(true))) + ) + ) + )) + } + } +} + +trait CandidatePushCopy { + self: PushCandidate => + + final lazy val pushCopy: MRPushCopy = + pushCopyId match { + case Some(pushCopyId) => + MrPushCopyObjects + .getCopyFromId(pushCopyId) + .getOrElse( + throw new InvalidPushCopyIdException( + s"Invalid push copy id: $pushCopyId for ${self.commonRecType}")) + + case None => + throw new PushCopyIdNotFoundException( + s"PushCopy not found in frigateNotification for ${self.commonRecType}" + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.scala new file mode 100644 index 0000000000..8e927254ad --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/InlineActionIbis2Hydrator.scala @@ -0,0 +1,12 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.util.InlineActionUtil +import com.twitter.util.Future + +trait InlineActionIbis2Hydrator { + self: PushCandidate => + + lazy val tweetInlineActionModelValue: Future[Map[String, String]] = + InlineActionUtil.getTweetInlineActionValue(target) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.scala new file mode 100644 index 0000000000..57483c8bac --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ListIbis2Hydrator.scala @@ -0,0 +1,21 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.pushservice.model.ListRecommendationPushCandidate +import com.twitter.util.Future + +trait ListIbis2Hydrator extends Ibis2HydratorForCandidate { + self: ListRecommendationPushCandidate => + + override lazy val senderId: Option[Long] = Some(0L) + + override lazy val modelValues: Future[Map[String, String]] = + Future.join(listName, listOwnerId).map { + case (nameOpt, authorId) => + Map( + "list" -> listId.toString, + "list_name" -> nameOpt + .getOrElse(""), + "list_author" -> s"${authorId.getOrElse(0L)}" + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.scala new file mode 100644 index 0000000000..edb0aa51e9 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutCreatorEventIbis2Hydrator.scala @@ -0,0 +1,29 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutCreatorEventPushCandidate +import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues +import com.twitter.util.Future + +trait MagicFanoutCreatorEventIbis2Hydrator + extends CustomConfigurationMapForIbis + with Ibis2HydratorForCandidate { + self: PushCandidate with MagicFanoutCreatorEventPushCandidate => + + val userMap = Map( + "handle" -> userProfile.screenName, + "display_name" -> userProfile.name + ) + + override val senderId = hydratedCreator.map(_.id) + + override lazy val modelValues: Future[Map[String, String]] = + mergeModelValues(super.modelValues, userMap) + + override val ibis2Request = creatorFanoutType match { + case CreatorFanoutType.UserSubscription => Future.None + case CreatorFanoutType.NewCreator => super.ibis2Request + case _ => super.ibis2Request + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.scala new file mode 100644 index 0000000000..a1b073a38e --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutNewsEventIbis2Hydrator.scala @@ -0,0 +1,103 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil +import com.twitter.frigate.pushservice.util.PushIbisUtil._ +import com.twitter.util.Future + +trait MagicFanoutNewsEventIbis2Hydrator extends Ibis2HydratorForCandidate { + self: PushCandidate with MagicFanoutEventHydratedCandidate => + + override lazy val senderId: Option[Long] = { + val isUgmMoment = self.semanticCoreEntityTags.values.flatten.toSet + .contains(MagicFanoutPredicatesUtil.UgmMomentTag) + + owningTwitterUserIds.headOption match { + case Some(owningTwitterUserId) + if isUgmMoment && target.params( + PushFeatureSwitchParams.MagicFanoutNewsUserGeneratedEventsEnable) => + Some(owningTwitterUserId) + case _ => None + } + } + + lazy val stats = self.statsReceiver.scope("MagicFanout") + lazy val defaultImageCounter = stats.counter("default_image") + lazy val requestImageCounter = stats.counter("request_num") + lazy val noneImageCounter = stats.counter("none_num") + + private def getModelValueMediaUrl( + urlOpt: Option[String], + mapKey: String + ): Option[(String, String)] = { + requestImageCounter.incr() + urlOpt match { + case Some(PushConstants.DefaultEventMediaUrl) => + defaultImageCounter.incr() + None + case Some(url) => Some(mapKey -> url) + case None => + noneImageCounter.incr() + None + } + } + + private lazy val eventModelValuesFut: Future[Map[String, String]] = { + for { + title <- eventTitleFut + squareImageUrl <- squareImageUrlFut + primaryImageUrl <- primaryImageUrlFut + eventDescriptionOpt <- eventDescriptionFut + } yield { + + val authorId = owningTwitterUserIds.headOption match { + case Some(author) + if target.params(PushFeatureSwitchParams.MagicFanoutNewsUserGeneratedEventsEnable) => + Some("author" -> author.toString) + case _ => None + } + + val eventDescription = eventDescriptionOpt match { + case Some(description) + if target.params(PushFeatureSwitchParams.MagicFanoutNewsEnableDescriptionCopy) => + Some("event_description" -> description) + case _ => + None + } + + Map( + "event_id" -> s"$eventId", + "event_title" -> title + ) ++ + getModelValueMediaUrl(squareImageUrl, "square_media_url") ++ + getModelValueMediaUrl(primaryImageUrl, "media_url") ++ + authorId ++ + eventDescription + } + } + + private lazy val topicValuesFut: Future[Map[String, String]] = { + if (target.params(PushFeatureSwitchParams.EnableTopicCopyForMF)) { + followedTopicLocalizedEntities.map(_.headOption).flatMap { + case Some(localizedEntity) => + Future.value(Map("topic_name" -> localizedEntity.localizedNameForDisplay)) + case _ => + ergLocalizedEntities.map(_.headOption).map { + case Some(localizedEntity) + if target.params(PushFeatureSwitchParams.EnableTopicCopyForImplicitTopics) => + Map("topic_name" -> localizedEntity.localizedNameForDisplay) + case _ => Map.empty[String, String] + } + } + } else { + Future.value(Map.empty[String, String]) + } + } + + override lazy val modelValues: Future[Map[String, String]] = + mergeFutModelValues(super.modelValues, mergeFutModelValues(eventModelValuesFut, topicValuesFut)) + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.scala new file mode 100644 index 0000000000..3062a66d09 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutProductLaunchIbis2Hydrator.scala @@ -0,0 +1,54 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate +import com.twitter.frigate.magic_events.thriftscala.ProductInfo +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues +import com.twitter.util.Future + +trait MagicFanoutProductLaunchIbis2Hydrator + extends CustomConfigurationMapForIbis + with Ibis2HydratorForCandidate { + self: PushCandidate with MagicFanoutProductLaunchCandidate => + + private def getProductInfoMap(productInfo: ProductInfo): Map[String, String] = { + val titleMap = productInfo.title + .map { title => + Map("title" -> title) + }.getOrElse(Map.empty) + val bodyMap = productInfo.body + .map { body => + Map("body" -> body) + }.getOrElse(Map.empty) + val deeplinkMap = productInfo.deeplink + .map { deeplink => + Map("deeplink" -> deeplink) + }.getOrElse(Map.empty) + + titleMap ++ bodyMap ++ deeplinkMap + } + + private lazy val landingPage: Map[String, String] = { + val urlFromFS = target.params(PushFeatureSwitchParams.ProductLaunchLandingPageDeepLink) + Map("push_land_url" -> urlFromFS) + } + + private lazy val customProductLaunchPushDetails: Map[String, String] = { + frigateNotification.magicFanoutProductLaunchNotification match { + case Some(productLaunchNotif) => + productLaunchNotif.productInfo match { + case Some(productInfo) => + getProductInfoMap(productInfo) + case _ => Map.empty + } + case _ => Map.empty + } + } + + override lazy val customFieldsMapFut: Future[Map[String, String]] = + mergeModelValues(super.customFieldsMapFut, customProductLaunchPushDetails) + + override lazy val modelValues: Future[Map[String, String]] = + mergeModelValues(super.modelValues, landingPage) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.scala new file mode 100644 index 0000000000..811caa9936 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/MagicFanoutSportsEventIbis2Hydrator.scala @@ -0,0 +1,89 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.common.base.BaseGameScore +import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate +import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation +import com.twitter.frigate.common.base.NflGameScore +import com.twitter.frigate.common.base.SoccerGameScore +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutSportsUtil +import com.twitter.frigate.pushservice.util.PushIbisUtil._ +import com.twitter.util.Future + +trait MagicFanoutSportsEventIbis2Hydrator extends Ibis2HydratorForCandidate { + self: PushCandidate + with MagicFanoutEventHydratedCandidate + with MagicFanoutSportsEventCandidate + with MagicFanoutSportsScoreInformation => + + lazy val stats = self.statsReceiver.scope("MagicFanoutSportsEvent") + lazy val defaultImageCounter = stats.counter("default_image") + lazy val requestImageCounter = stats.counter("request_num") + lazy val noneImageCounter = stats.counter("none_num") + + override lazy val relevanceScoreMapFut = Future.value(Map.empty[String, String]) + + private def getModelValueMediaUrl( + urlOpt: Option[String], + mapKey: String + ): Option[(String, String)] = { + requestImageCounter.incr() + urlOpt match { + case Some(PushConstants.DefaultEventMediaUrl) => + defaultImageCounter.incr() + None + case Some(url) => Some(mapKey -> url) + case None => + noneImageCounter.incr() + None + } + } + + private lazy val eventModelValuesFut: Future[Map[String, String]] = { + for { + title <- eventTitleFut + squareImageUrl <- squareImageUrlFut + primaryImageUrl <- primaryImageUrlFut + } yield { + Map( + "event_id" -> s"$eventId", + "event_title" -> title + ) ++ + getModelValueMediaUrl(squareImageUrl, "square_media_url") ++ + getModelValueMediaUrl(primaryImageUrl, "media_url") + } + } + + private lazy val sportsScoreValues: Future[Map[String, String]] = { + for { + scores <- gameScores + homeName <- homeTeamInfo.map(_.map(_.name)) + awayName <- awayTeamInfo.map(_.map(_.name)) + } yield { + if (awayName.isDefined && homeName.isDefined && scores.isDefined) { + scores.get match { + case game: SoccerGameScore => + MagicFanoutSportsUtil.getSoccerIbisMap(game) ++ Map( + "away_team" -> awayName.get, + "home_team" -> homeName.get + ) + case game: NflGameScore => + MagicFanoutSportsUtil.getNflIbisMap(game) ++ Map( + "away_team" -> MagicFanoutSportsUtil.getNFLReadableName(awayName.get), + "home_team" -> MagicFanoutSportsUtil.getNFLReadableName(homeName.get) + ) + case baseGameScore: BaseGameScore => + Map.empty[String, String] + } + } else Map.empty[String, String] + } + } + + override lazy val customFieldsMapFut: Future[Map[String, String]] = + mergeFutModelValues(super.customFieldsMapFut, sportsScoreValues) + + override lazy val modelValues: Future[Map[String, String]] = + mergeFutModelValues(super.modelValues, eventModelValuesFut) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.scala new file mode 100644 index 0000000000..c7bd051b79 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OutOfNetworkTweetIbis2Hydrator.scala @@ -0,0 +1,90 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate +import com.twitter.frigate.common.base.TopicCandidate +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.rec_types.RecTypes._ +import com.twitter.frigate.common.util.MrPushCopyObjects +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.util.InlineActionUtil +import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.util.Future + +trait OutOfNetworkTweetIbis2HydratorForCandidate extends TweetCandidateIbis2Hydrator { + self: PushCandidate with OutOfNetworkTweetCandidate with TopicCandidate with TweetAuthorDetails => + + private lazy val useNewOonCopyValue = + if (target.params(PushFeatureSwitchParams.EnableNewMROONCopyForPush)) { + Map( + "use_new_oon_copy" -> "true" + ) + } else Map.empty[String, String] + + override lazy val tweetDynamicInlineActionsModelValues = + if (target.params(PushFeatureSwitchParams.EnableOONGeneratedInlineActions)) { + val actions = target.params(PushFeatureSwitchParams.OONTweetDynamicInlineActionsList) + InlineActionUtil.getGeneratedTweetInlineActions(target, statsReceiver, actions) + } else Map.empty[String, String] + + private lazy val ibtModelValues: Map[String, String] = + Map( + "is_tweet" -> s"${!(hasPhoto || hasVideo)}", + "is_photo" -> s"$hasPhoto", + "is_video" -> s"$hasVideo" + ) + + private lazy val launchVideosInImmersiveExploreValue = + Map( + "launch_videos_in_immersive_explore" -> s"${hasVideo && target.params(PushFeatureSwitchParams.EnableLaunchVideosInImmersiveExplore)}" + ) + + private lazy val oonTweetModelValues = + useNewOonCopyValue ++ ibtModelValues ++ tweetDynamicInlineActionsModelValues ++ launchVideosInImmersiveExploreValue + + lazy val useTopicCopyForMBCGIbis = mrModelingBasedTypes.contains(commonRecType) && target.params( + PushFeatureSwitchParams.EnableMrModelingBasedCandidatesTopicCopy) + lazy val useTopicCopyForFrsIbis = frsTypes.contains(commonRecType) && target.params( + PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicCopy) + lazy val useTopicCopyForTagspaceIbis = tagspaceTypes.contains(commonRecType) && target.params( + PushFeatureSwitchParams.EnableHashspaceCandidatesTopicCopy) + + override lazy val modelName: String = { + if (localizedUttEntity.isDefined && + (useTopicCopyForMBCGIbis || useTopicCopyForFrsIbis || useTopicCopyForTagspaceIbis)) { + MrPushCopyObjects.TopicTweet.ibisPushModelName // uses topic copy + } else super.modelName + } + + lazy val exploreVideoParams: Map[String, String] = { + if (self.commonRecType == CommonRecommendationType.ExploreVideoTweet) { + Map( + "is_explore_video" -> "true" + ) + } else Map.empty[String, String] + } + + override lazy val customFieldsMapFut: Future[Map[String, String]] = + mergeModelValues(super.customFieldsMapFut, exploreVideoParams) + + override lazy val tweetModelValues: Future[Map[String, String]] = + if (localizedUttEntity.isDefined && + (useTopicCopyForMBCGIbis || useTopicCopyForFrsIbis || useTopicCopyForTagspaceIbis)) { + lazy val topicTweetModelValues: Map[String, String] = + Map("topic_name" -> s"${localizedUttEntity.get.localizedNameForDisplay}") + for { + superModelValues <- super.tweetModelValues + tweetInlineModelValue <- tweetInlineActionModelValue + } yield { + superModelValues ++ topicTweetModelValues ++ tweetInlineModelValue + } + } else { + for { + superModelValues <- super.tweetModelValues + tweetInlineModelValues <- tweetInlineActionModelValue + } yield { + superModelValues ++ mediaModelValue ++ oonTweetModelValues ++ tweetInlineModelValues ++ inlineVideoMediaMap + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.scala new file mode 100644 index 0000000000..e802a64212 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/OverrideForIbis2Request.scala @@ -0,0 +1,210 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.common.store.deviceinfo.DeviceInfo +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} +import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.ContinuousFunction +import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.ContinuousFunctionParam +import com.twitter.frigate.pushservice.util.OverrideNotificationUtil +import com.twitter.frigate.pushservice.util.PushCapUtil +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.thriftscala.CommonRecommendationType.MagicFanoutSportsEvent +import com.twitter.ibis2.lib.util.JsonMarshal +import com.twitter.util.Future + +trait OverrideForIbis2Request { + self: PushCandidate => + + private lazy val overrideStats = self.statsReceiver.scope("override_for_ibis2") + + private lazy val addedOverrideAndroidCounter = + overrideStats.scope("android").counter("added_override_for_ibis2_request") + private lazy val addedSmartPushConfigAndroidCounter = + overrideStats.scope("android").counter("added_smart_push_config_for_ibis2_request") + private lazy val addedOverrideIosCounter = + overrideStats.scope("ios").counter("added_override_for_ibis2_request") + private lazy val noOverrideCounter = overrideStats.counter("no_override_for_ibis2_request") + private lazy val noOverrideDueToDeviceInfoCounter = + overrideStats.counter("no_override_due_to_device_info") + private lazy val addedMlScoreToPayloadAndroid = + overrideStats.scope("android").counter("added_ml_score") + private lazy val noMlScoreAddedToPayload = + overrideStats.counter("no_ml_score") + private lazy val addedNSlotsToPayload = + overrideStats.counter("added_n_slots") + private lazy val noNSlotsAddedToPayload = + overrideStats.counter("no_n_slots") + private lazy val addedCustomThreadIdToPayload = + overrideStats.counter("added_custom_thread_id") + private lazy val noCustomThreadIdAddedToPayload = + overrideStats.counter("no_custom_thread_id") + private lazy val enableTargetIdOverrideForMagicFanoutSportsEventCounter = + overrideStats.counter("enable_target_id_override_for_mf_sports_event") + + lazy val candidateModelScoreFut: Future[Option[Double]] = { + if (RecTypes.notEligibleForModelScoreTracking(commonRecType)) Future.None + else mrWeightedOpenOrNtabClickRankingProbability + } + + lazy val overrideModelValueFut: Future[Map[String, String]] = { + if (self.target.isLoggedOutUser) { + Future.value(Map.empty[String, String]) + } else { + Future + .join( + target.deviceInfo, + target.accountCountryCode, + OverrideNotificationUtil.getCollapseAndImpressionIdForOverride(self), + candidateModelScoreFut, + target.dynamicPushcap, + target.optoutAdjustedPushcap, + PushCapUtil.getDefaultPushCap(target) + ).map { + case ( + deviceInfoOpt, + countryCodeOpt, + Some((collapseId, impressionIds)), + mlScore, + dynamicPushcapOpt, + optoutAdjustedPushcapOpt, + defaultPushCap) => + val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { + case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap + case (Some(pushcapInfo), _) => pushcapInfo.pushcap + case _ => defaultPushCap + } + getClientSpecificOverrideModelValues( + target, + deviceInfoOpt, + countryCodeOpt, + collapseId, + impressionIds, + mlScore, + pushCap) + case _ => + noOverrideCounter.incr() + Map.empty[String, String] + } + } + } + + /** + * Determines the appropriate Override Notification model values based on the client + * @param target Target that will be receiving the push recommendation + * @param deviceInfoOpt Target's Device Info + * @param collapseId Collapse ID determined by OverrideNotificationUtil + * @param impressionIds Impression IDs of previously sent Override Notifications + * @param mlScore Open/NTab click ranking score of the current push candidate + * @param pushCap Push cap for the target + * @return Map consisting of the model values that need to be added to the Ibis2 Request + */ + def getClientSpecificOverrideModelValues( + target: Target, + deviceInfoOpt: Option[DeviceInfo], + countryCodeOpt: Option[String], + collapseId: String, + impressionIds: Seq[String], + mlScoreOpt: Option[Double], + pushCap: Int + ): Map[String, String] = { + + val primaryDeviceIos = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) + val primaryDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) + + if (primaryDeviceIos || + (primaryDeviceAndroid && + target.params(FSParams.EnableOverrideNotificationsSmartPushConfigForAndroid))) { + + if (primaryDeviceIos) addedOverrideIosCounter.incr() + else addedSmartPushConfigAndroidCounter.incr() + + val impressionIdsSeq = { + if (target.params(FSParams.EnableTargetIdsInSmartPushPayload)) { + if (target.params(FSParams.EnableOverrideNotificationsMultipleTargetIds)) + impressionIds + else Seq(impressionIds.head) + } + // Explicitly enable targetId-based override for MagicFanoutSportsEvent candidates (live sport update notifications) + else if (self.commonRecType == MagicFanoutSportsEvent && target.params( + FSParams.EnableTargetIdInSmartPushPayloadForMagicFanoutSportsEvent)) { + enableTargetIdOverrideForMagicFanoutSportsEventCounter.incr() + Seq(impressionIds.head) + } else Seq.empty[String] + } + + val mlScoreMap = mlScoreOpt match { + case Some(mlScore) + if target.params(FSParams.EnableOverrideNotificationsScoreBasedOverride) => + addedMlScoreToPayloadAndroid.incr() + Map("score" -> mlScore) + case _ => + noMlScoreAddedToPayload.incr() + Map.empty + } + + val nSlotsMap = { + if (target.params(FSParams.EnableOverrideNotificationsNSlots)) { + if (target.params(FSParams.EnableOverrideMaxSlotFn)) { + val nslotFnParam = ContinuousFunctionParam( + target + .params(PushFeatureSwitchParams.OverrideMaxSlotFnPushCapKnobs), + target + .params(PushFeatureSwitchParams.OverrideMaxSlotFnNSlotKnobs), + target + .params(PushFeatureSwitchParams.OverrideMaxSlotFnPowerKnobs), + target + .params(PushFeatureSwitchParams.OverrideMaxSlotFnWeight), + target.params(FSParams.OverrideNotificationsMaxNumOfSlots) + ) + val numOfSlots = ContinuousFunction.safeEvaluateFn( + pushCap, + nslotFnParam, + overrideStats.scope("max_nslot_fn")) + overrideStats.counter("max_notification_slots_num_" + numOfSlots.toString).incr() + addedNSlotsToPayload.incr() + Map("max_notification_slots" -> numOfSlots) + } else { + addedNSlotsToPayload.incr() + val numOfSlots = target.params(FSParams.OverrideNotificationsMaxNumOfSlots) + Map("max_notification_slots" -> numOfSlots) + } + } else { + noNSlotsAddedToPayload.incr() + Map.empty + } + } + + val baseActionDetailsMap = Map("target_ids" -> impressionIdsSeq) + + val actionDetailsMap = + Map("action_details" -> (baseActionDetailsMap ++ nSlotsMap)) + + val baseSmartPushConfigMap = Map("notification_action" -> "REPLACE") + + val customThreadId = { + if (target.params(FSParams.EnableCustomThreadIdForOverride)) { + addedCustomThreadIdToPayload.incr() + Map("custom_thread_id" -> impressionId) + } else { + noCustomThreadIdAddedToPayload.incr() + Map.empty + } + } + + val smartPushConfigMap = + JsonMarshal.toJson( + baseSmartPushConfigMap ++ actionDetailsMap ++ mlScoreMap ++ customThreadId) + + Map("smart_notification_configuration" -> smartPushConfigMap) + } else if (primaryDeviceAndroid) { + addedOverrideAndroidCounter.incr() + Map("notification_id" -> collapseId, "overriding_impression_id" -> impressionIds.head) + } else { + noOverrideDueToDeviceInfoCounter.incr() + Map.empty[String, String] + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.scala new file mode 100644 index 0000000000..359b876a1b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/PushOverrideInfo.scala @@ -0,0 +1,246 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.history.History +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.frigate.thriftscala.OverrideInfo +import com.twitter.util.Duration +import com.twitter.util.Time + +object PushOverrideInfo { + + private val name: String = this.getClass.getSimpleName + + /** + * Gets all eligible time + override push notification pairs from a target's History + * + * @param history: history of push notifications + * @param lookbackDuration: duration to look back up in history for overriding notifications + * @return: list of notifications with send timestamps which are eligible for overriding + */ + def getOverrideEligibleHistory( + history: History, + lookbackDuration: Duration, + ): Seq[(Time, FrigateNotification)] = { + history.sortedHistory + .takeWhile { case (notifTimestamp, _) => lookbackDuration.ago < notifTimestamp } + .filter { + case (_, notification) => notification.overrideInfo.isDefined + } + } + + /** + * Gets all eligible override push notifications from a target's History + * + * @param history Target's History + * @param lookbackDuration Duration in which we would like to obtain the eligible push notifications + * @param stats StatsReceiver to track stats for this function + * @return Returns a list of FrigateNotification + */ + def getOverrideEligiblePushNotifications( + history: History, + lookbackDuration: Duration, + stats: StatsReceiver, + ): Seq[FrigateNotification] = { + val eligibleNotificationsDistribution = + stats.scope(name).stat("eligible_notifications_size_distribution") + val eligibleNotificationsSeq = + getOverrideEligibleHistory(history, lookbackDuration) + .collect { + case (_, notification) => notification + } + + eligibleNotificationsDistribution.add(eligibleNotificationsSeq.size) + eligibleNotificationsSeq + } + + /** + * Gets the OverrideInfo for the last eligible Override Notification FrigateNotification, if it exists + * @param history Target's History + * @param lookbackDuration Duration in which we would like to obtain the last override notification + * @param stats StatsReceiver to track stats for this function + * @return Returns OverrideInfo of the last MR push, else None + */ + def getOverrideInfoOfLastEligiblePushNotif( + history: History, + lookbackDuration: Duration, + stats: StatsReceiver + ): Option[OverrideInfo] = { + val overrideInfoEmptyOfLastPush = stats.scope(name).counter("override_info_empty_of_last_push") + val overrideInfoExistsForLastPush = + stats.scope(name).counter("override_info_exists_for_last_push") + val overrideHistory = + getOverrideEligiblePushNotifications(history, lookbackDuration, stats) + if (overrideHistory.isEmpty) { + overrideInfoEmptyOfLastPush.incr() + None + } else { + overrideInfoExistsForLastPush.incr() + overrideHistory.head.overrideInfo + } + } + + /** + * Gets all the MR Push Notifications in the specified override chain + * @param history Target's History + * @param overrideChainId Override Chain Identifier + * @param stats StatsReceiver to track stats for this function + * @return Returns a sequence of FrigateNotification that exist in the override chain + */ + def getMrPushNotificationsInOverrideChain( + history: History, + overrideChainId: String, + stats: StatsReceiver + ): Seq[FrigateNotification] = { + val notificationInOverrideChain = stats.scope(name).counter("notification_in_override_chain") + val notificationNotInOverrideChain = + stats.scope(name).counter("notification_not_in_override_chain") + history.sortedHistory.flatMap { + case (_, notification) + if isNotificationInOverrideChain(notification, overrideChainId, stats) => + notificationInOverrideChain.incr() + Some(notification) + case _ => + notificationNotInOverrideChain.incr() + None + } + } + + /** + * Gets the timestamp (in milliseconds) for the specified FrigateNotification + * @param notification The FrigateNotification that we would like the timestamp for + * @param history Target's History + * @param stats StatsReceiver to track stats for this function + * @return Returns the timestamp in milliseconds for the specified notification + * if it exists History, else None + */ + def getTimestampInMillisForFrigateNotification( + notification: FrigateNotification, + history: History, + stats: StatsReceiver + ): Option[Long] = { + val foundTimestampOfNotificationInHistory = + stats.scope(name).counter("found_timestamp_of_notification_in_history") + history.sortedHistory + .find(_._2.equals(notification)).map { + case (time, _) => + foundTimestampOfNotificationInHistory.incr() + time.inMilliseconds + } + } + + /** + * Gets the oldest frigate notification based on the user's NTab last read position + * @param overrideCandidatesMap All the NTab Notifications in the override chain + * @return Returns the oldest frigate notification in the chain + */ + def getOldestFrigateNotification( + overrideCandidatesMap: Map[Long, FrigateNotification], + ): FrigateNotification = { + overrideCandidatesMap.minBy(_._1)._2 + } + + /** + * Gets the impression ids of previous eligible push notification. + * @param history Target's History + * @param lookbackDuration Duration in which we would like to obtain previous impression ids + * @param stats StatsReceiver to track stats for this function + * @return Returns the impression identifier for the last eligible push notif. + * if it exists in the target's History, else None. + */ + def getImpressionIdsOfPrevEligiblePushNotif( + history: History, + lookbackDuration: Duration, + stats: StatsReceiver + ): Seq[String] = { + val foundImpressionIdOfLastEligiblePushNotif = + stats.scope(name).counter("found_impression_id_of_last_eligible_push_notif") + val overrideHistoryEmptyWhenFetchingImpressionId = + stats.scope(name).counter("override_history_empty_when_fetching_impression_id") + val overrideHistory = getOverrideEligiblePushNotifications(history, lookbackDuration, stats) + .filter(frigateNotification => + // Exclude notifications of nonGenericOverrideTypes from being overridden + !RecTypes.nonGenericOverrideTypes.contains(frigateNotification.commonRecommendationType)) + + if (overrideHistory.isEmpty) { + overrideHistoryEmptyWhenFetchingImpressionId.incr() + Seq.empty + } else { + foundImpressionIdOfLastEligiblePushNotif.incr() + overrideHistory.flatMap(_.impressionId) + } + } + + /** + * Gets the impressions ids by eventId, for MagicFanoutEvent candidates. + * + * @param history Target's History + * @param lookbackDuration Duration in which we would like to obtain previous impression ids + * @param stats StatsReceiver to track stats for this function + * @param overridableType Specific MagicFanoutEvent CRT + * @param eventId Event identifier for MagicFanoutEventCandidate. + * @return Returns the impression identifiers for the last eligible, eventId-matching + * MagicFanoutEvent push notifications if they exist in the target's history, else None. + */ + def getImpressionIdsForPrevEligibleMagicFanoutEventCandidates( + history: History, + lookbackDuration: Duration, + stats: StatsReceiver, + overridableType: CommonRecommendationType, + eventId: Long + ): Seq[String] = { + val foundImpressionIdOfMagicFanoutEventNotif = + stats.scope(name).counter("found_impression_id_of_magic_fanout_event_notif") + val overrideHistoryEmptyWhenFetchingImpressionId = + stats + .scope(name).counter( + "override_history_empty_when_fetching_impression_id_for_magic_fanout_event_notif") + + val overrideHistory = + getOverrideEligiblePushNotifications(history, lookbackDuration, stats) + .filter(frigateNotification => + // Only override notifications with same CRT and eventId + frigateNotification.commonRecommendationType == overridableType && + frigateNotification.magicFanoutEventNotification.exists(_.eventId == eventId)) + + if (overrideHistory.isEmpty) { + overrideHistoryEmptyWhenFetchingImpressionId.incr() + Seq.empty + } else { + foundImpressionIdOfMagicFanoutEventNotif.incr() + overrideHistory.flatMap(_.impressionId) + } + } + + /** + * Determines if the provided notification is part of the specified override chain + * @param notification FrigateNotification that we're trying to identify as within the override chain + * @param overrideChainId Override Chain Identifier + * @param stats StatsReceiver to track stats for this function + * @return Returns true if the provided FrigateNotification is within the override chain, else false + */ + private def isNotificationInOverrideChain( + notification: FrigateNotification, + overrideChainId: String, + stats: StatsReceiver + ): Boolean = { + val notifIsInOverrideChain = stats.scope(name).counter("notif_is_in_override_chain") + val notifNotInOverrideChain = stats.scope(name).counter("notif_not_in_override_chain") + notification.overrideInfo match { + case Some(overrideInfo) => + val isNotifInOverrideChain = overrideInfo.collapseInfo.overrideChainId == overrideChainId + if (isNotifInOverrideChain) { + notifIsInOverrideChain.incr() + true + } else { + notifNotInOverrideChain.incr() + false + } + case _ => + notifNotInOverrideChain.incr() + false + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.scala new file mode 100644 index 0000000000..479c230eb4 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/RankedSocialContextIbis2Hydrator.scala @@ -0,0 +1,22 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.common.base.SocialContextAction +import com.twitter.frigate.common.base.SocialContextActions +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.frigate.pushservice.util.PushIbisUtil +import com.twitter.util.Future + +trait RankedSocialContextIbis2Hydrator { + self: PushCandidate with SocialContextActions => + + lazy val socialContextModelValues: Future[Map[String, String]] = + rankedSocialContextActionsFut.map(rankedSocialContextActions => + PushIbisUtil.getSocialContextModelValues(rankedSocialContextActions.map(_.userId))) + + lazy val rankedSocialContextActionsFut: Future[Seq[SocialContextAction]] = + CandidateUtil.getRankedSocialContext( + socialContextActions, + target.seedsWithWeight, + defaultToRecency = false) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.scala new file mode 100644 index 0000000000..d1a4399721 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSpeakerIbis2Hydrator.scala @@ -0,0 +1,34 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.pushservice.model.ScheduledSpaceSpeakerPushCandidate +import com.twitter.frigate.pushservice.util.PushIbisUtil._ +import com.twitter.frigate.thriftscala.SpaceNotificationType +import com.twitter.util.Future + +trait ScheduledSpaceSpeakerIbis2Hydrator extends Ibis2HydratorForCandidate { + self: ScheduledSpaceSpeakerPushCandidate => + + override lazy val senderId: Option[Long] = None + + private lazy val targetModelValues: Future[Map[String, String]] = { + hostId match { + case Some(spaceHostId) => + audioSpaceFut.map { audioSpace => + val isStartNow = frigateNotification.spaceNotification.exists( + _.spaceNotificationType.contains(SpaceNotificationType.AtSpaceBroadcast)) + + Map( + "host_id" -> s"$spaceHostId", + "space_id" -> spaceId, + "is_start_now" -> s"$isStartNow" + ) ++ audioSpace.flatMap(_.title.map("space_title" -> _)) + } + case _ => + Future.exception( + new IllegalStateException("Unable to get host id for ScheduledSpaceSpeakerIbis2Hydrator")) + } + } + + override lazy val modelValues: Future[Map[String, String]] = + mergeFutModelValues(super.modelValues, targetModelValues) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.scala new file mode 100644 index 0000000000..b1486de3f3 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/ScheduledSpaceSubscriberIbis2Hydrator.scala @@ -0,0 +1,29 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.pushservice.model.ScheduledSpaceSubscriberPushCandidate +import com.twitter.frigate.pushservice.util.PushIbisUtil._ +import com.twitter.util.Future + +trait ScheduledSpaceSubscriberIbis2Hydrator extends Ibis2HydratorForCandidate { + self: ScheduledSpaceSubscriberPushCandidate => + + override lazy val senderId: Option[Long] = hostId + + private lazy val targetModelValues: Future[Map[String, String]] = { + hostId match { + case Some(spaceHostId) => + audioSpaceFut.map { audioSpace => + Map( + "host_id" -> s"$spaceHostId", + "space_id" -> spaceId, + ) ++ audioSpace.flatMap(_.title.map("space_title" -> _)) + } + case _ => + Future.exception( + new RuntimeException("Unable to get host id for ScheduledSpaceSubscriberIbis2Hydrator")) + } + } + + override lazy val modelValues: Future[Map[String, String]] = + mergeFutModelValues(super.modelValues, targetModelValues) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.scala new file mode 100644 index 0000000000..a61edc5094 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/SubscribedSearchTweetIbis2Hydrator.scala @@ -0,0 +1,33 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.pushservice.model.SubscribedSearchTweetPushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.util.InlineActionUtil +import com.twitter.util.Future + +trait SubscribedSearchTweetIbis2Hydrator extends TweetCandidateIbis2Hydrator { + self: SubscribedSearchTweetPushCandidate => + + override lazy val tweetDynamicInlineActionsModelValues = { + if (target.params(PushFeatureSwitchParams.EnableOONGeneratedInlineActions)) { + val actions = target.params(PushFeatureSwitchParams.TweetDynamicInlineActionsList) + InlineActionUtil.getGeneratedTweetInlineActions(target, statsReceiver, actions) + } else Map.empty[String, String] + } + + private lazy val searchTermValue: Map[String, String] = + Map( + "search_term" -> searchTerm, + "search_url" -> pushLandingUrl + ) + + private lazy val searchModelValues = searchTermValue ++ tweetDynamicInlineActionsModelValues + + override lazy val tweetModelValues: Future[Map[String, String]] = + for { + superModelValues <- super.tweetModelValues + tweetInlineModelValues <- tweetInlineActionModelValue + } yield { + superModelValues ++ mediaModelValue ++ searchModelValues ++ tweetInlineModelValues ++ inlineVideoMediaMap + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.scala new file mode 100644 index 0000000000..e12733fb21 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopTweetImpressionsCandidateIbis2Hydrator.scala @@ -0,0 +1,21 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.common.base.TopTweetImpressionsCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeFutModelValues +import com.twitter.util.Future + +trait TopTweetImpressionsCandidateIbis2Hydrator extends Ibis2HydratorForCandidate { + self: PushCandidate with TopTweetImpressionsCandidate => + + private lazy val targetModelValues: Map[String, String] = { + Map( + "target_user" -> target.targetId.toString, + "tweet" -> tweetId.toString, + "impressions_count" -> impressionsCount.toString + ) + } + + override lazy val modelValues: Future[Map[String, String]] = + mergeFutModelValues(super.modelValues, Future.value(targetModelValues)) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.scala new file mode 100644 index 0000000000..6a187dfeb3 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TopicProofTweetIbis2Hydrator.scala @@ -0,0 +1,32 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.pushservice.model.TopicProofTweetPushCandidate +import com.twitter.frigate.pushservice.exception.UttEntityNotFoundException +import com.twitter.util.Future + +trait TopicProofTweetIbis2Hydrator extends TweetCandidateIbis2Hydrator { + self: TopicProofTweetPushCandidate => + + private lazy val implicitTopicTweetModelValues: Map[String, String] = { + val uttEntity = localizedUttEntity.getOrElse( + throw new UttEntityNotFoundException( + s"${getClass.getSimpleName} UttEntity missing for $tweetId")) + + Map( + "topic_name" -> uttEntity.localizedNameForDisplay, + "topic_id" -> uttEntity.entityId.toString + ) + } + + override lazy val modelName: String = pushCopy.ibisPushModelName + + override lazy val tweetModelValues: Future[Map[String, String]] = + for { + superModelValues <- super.tweetModelValues + tweetInlineModelValues <- tweetInlineActionModelValue + } yield { + superModelValues ++ + tweetInlineModelValues ++ + implicitTopicTweetModelValues + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.scala new file mode 100644 index 0000000000..1c3420df49 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TrendTweetIbis2Hydrator.scala @@ -0,0 +1,16 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.common.base.TrendTweetCandidate +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate + +trait TrendTweetIbis2Hydrator extends TweetCandidateIbis2Hydrator { + self: PushCandidate with TrendTweetCandidate with TweetAuthorDetails => + + lazy val trendNameModelValue = Map("trend_name" -> trendName) + + override lazy val tweetModelValues = for { + tweetValues <- super.tweetModelValues + inlineActionValues <- tweetInlineActionModelValue + } yield tweetValues ++ inlineActionValues ++ trendNameModelValue +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.scala new file mode 100644 index 0000000000..0b0a5db054 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetCandidateIbis2Hydrator.scala @@ -0,0 +1,166 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.base.TweetDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.SubtextForAndroidPushHeader +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} +import com.twitter.frigate.pushservice.util.CopyUtil +import com.twitter.frigate.pushservice.util.EmailLandingPageExperimentUtil +import com.twitter.frigate.pushservice.util.InlineActionUtil +import com.twitter.frigate.pushservice.util.PushToHomeUtil +import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeFutModelValues +import com.twitter.util.Future + +trait TweetCandidateIbis2Hydrator + extends Ibis2HydratorForCandidate + with InlineActionIbis2Hydrator + with CustomConfigurationMapForIbis { + self: PushCandidate with TweetCandidate with TweetDetails with TweetAuthorDetails => + + lazy val scopedStats: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) + + lazy val tweetIdModelValue: Map[String, String] = + Map( + "tweet" -> tweetId.toString + ) + + lazy val authorModelValue: Map[String, String] = { + assert(authorId.isDefined) + Map( + "author" -> authorId.getOrElse(0L).toString + ) + } + + lazy val otherModelValues: Map[String, String] = + Map( + "show_explanatory_text" -> "true", + "show_negative_feedback" -> "true" + ) + + lazy val mediaModelValue: Map[String, String] = + Map( + "show_media" -> "true" + ) + + lazy val inlineVideoMediaMap: Map[String, String] = { + if (hasVideo) { + val isInlineVideoEnabled = target.params(FS.EnableInlineVideo) + val isAutoplayEnabled = target.params(FS.EnableAutoplayForInlineVideo) + Map( + "enable_inline_video_for_ios" -> isInlineVideoEnabled.toString, + "enable_autoplay_for_inline_video_ios" -> isAutoplayEnabled.toString + ) + } else Map.empty + } + + lazy val landingPageModelValues: Future[Map[String, String]] = { + for { + deviceInfoOpt <- target.deviceInfo + } yield { + PushToHomeUtil.getIbis2ModelValue(deviceInfoOpt, target, scopedStats) match { + case Some(pushToHomeModelValues) => pushToHomeModelValues + case _ => + EmailLandingPageExperimentUtil.getIbis2ModelValue( + deviceInfoOpt, + target, + tweetId + ) + } + } + } + + lazy val tweetDynamicInlineActionsModelValues = { + if (target.params(PushFeatureSwitchParams.EnableTweetDynamicInlineActions)) { + val actions = target.params(PushFeatureSwitchParams.TweetDynamicInlineActionsList) + InlineActionUtil.getGeneratedTweetInlineActions(target, statsReceiver, actions) + } else Map.empty[String, String] + } + + lazy val tweetDynamicInlineActionsModelValuesForWeb: Map[String, String] = { + if (target.isLoggedOutUser) { + Map.empty[String, String] + } else { + InlineActionUtil.getGeneratedTweetInlineActionsForWeb( + actions = target.params(PushFeatureSwitchParams.TweetDynamicInlineActionsListForWeb), + enableForDesktopWeb = + target.params(PushFeatureSwitchParams.EnableDynamicInlineActionsForDesktopWeb), + enableForMobileWeb = + target.params(PushFeatureSwitchParams.EnableDynamicInlineActionsForMobileWeb) + ) + } + } + + lazy val copyFeaturesFut: Future[Map[String, String]] = + CopyUtil.getCopyFeatures(self, scopedStats) + + private def getVerifiedSymbolModelValue: Future[Map[String, String]] = { + self.tweetAuthor.map { + case Some(author) => + if (author.safety.exists(_.verified)) { + scopedStats.counter("is_verified").incr() + if (target.params(FS.EnablePushPresentationVerifiedSymbol)) { + scopedStats.counter("is_verified_and_add").incr() + Map("is_author_verified" -> "true") + } else { + scopedStats.counter("is_verified_and_NOT_add").incr() + Map.empty + } + } else { + scopedStats.counter("is_NOT_verified").incr() + Map.empty + } + case _ => + scopedStats.counter("none_author").incr() + Map.empty + } + } + + private def subtextAndroidPushHeader: Map[String, String] = { + self.target.params(PushFeatureSwitchParams.SubtextInAndroidPushHeaderParam) match { + case SubtextForAndroidPushHeader.None => + Map.empty + case SubtextForAndroidPushHeader.TargetHandler => + Map("subtext_target_handler" -> "true") + case SubtextForAndroidPushHeader.TargetTagHandler => + Map("subtext_target_tag_handler" -> "true") + case SubtextForAndroidPushHeader.TargetName => + Map("subtext_target_name" -> "true") + case SubtextForAndroidPushHeader.AuthorTagHandler => + Map("subtext_author_tag_handler" -> "true") + case SubtextForAndroidPushHeader.AuthorName => + Map("subtext_author_name" -> "true") + case _ => + Map.empty + } + } + + lazy val bodyPushMap: Map[String, String] = { + if (self.target.params(PushFeatureSwitchParams.EnableEmptyBody)) { + Map("enable_empty_body" -> "true") + } else Map.empty[String, String] + } + + override def customFieldsMapFut: Future[Map[String, String]] = + for { + superModelValues <- super.customFieldsMapFut + copyFeaturesModelValues <- copyFeaturesFut + verifiedSymbolModelValue <- getVerifiedSymbolModelValue + } yield { + superModelValues ++ copyFeaturesModelValues ++ + verifiedSymbolModelValue ++ subtextAndroidPushHeader ++ bodyPushMap + } + + override lazy val senderId: Option[Long] = authorId + + def tweetModelValues: Future[Map[String, String]] = + landingPageModelValues.map { landingPageModelValues => + tweetIdModelValue ++ authorModelValue ++ landingPageModelValues ++ tweetDynamicInlineActionsModelValues ++ tweetDynamicInlineActionsModelValuesForWeb + } + + override lazy val modelValues: Future[Map[String, String]] = + mergeFutModelValues(super.modelValues, tweetModelValues) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.scala new file mode 100644 index 0000000000..ae4cd91747 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetFavoriteIbis2Hydrator.scala @@ -0,0 +1,21 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetFavoriteCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.util.Future + +trait TweetFavoriteCandidateIbis2Hydrator + extends TweetCandidateIbis2Hydrator + with RankedSocialContextIbis2Hydrator { + self: PushCandidate with TweetFavoriteCandidate with TweetAuthorDetails => + + override lazy val tweetModelValues: Future[Map[String, String]] = + for { + socialContextModelValues <- socialContextModelValues + superModelValues <- super.tweetModelValues + tweetInlineModelValues <- tweetInlineActionModelValue + } yield { + superModelValues ++ mediaModelValue ++ otherModelValues ++ socialContextModelValues ++ tweetInlineModelValues + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.scala new file mode 100644 index 0000000000..2b665a8faf --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ibis/TweetRetweetIbis2Hydrator.scala @@ -0,0 +1,32 @@ +package com.twitter.frigate.pushservice.model.ibis + +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetRetweetCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.util.PushIbisUtil.mergeModelValues + +import com.twitter.util.Future + +trait TweetRetweetCandidateIbis2Hydrator + extends TweetCandidateIbis2Hydrator + with RankedSocialContextIbis2Hydrator { + self: PushCandidate with TweetRetweetCandidate with TweetAuthorDetails => + + override lazy val tweetModelValues: Future[Map[String, String]] = + for { + socialContextModelValues <- socialContextModelValues + superModelValues <- super.tweetModelValues + tweetInlineModelValues <- tweetInlineActionModelValue + } yield { + superModelValues ++ mediaModelValue ++ otherModelValues ++ socialContextModelValues ++ tweetInlineModelValues ++ inlineVideoMediaMap + } + + lazy val socialContextForRetweetMap: Map[String, String] = + if (self.target.params(PushFeatureSwitchParams.EnableSocialContextForRetweet)) { + Map("enable_social_context_retweet" -> "true") + } else Map.empty[String, String] + + override lazy val customFieldsMapFut: Future[Map[String, String]] = + mergeModelValues(super.customFieldsMapFut, socialContextForRetweetMap) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.scala new file mode 100644 index 0000000000..ef80db5b5f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/CandidateNTabCopy.scala @@ -0,0 +1,21 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.util.MRNtabCopy +import com.twitter.frigate.common.util.MrNtabCopyObjects +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.take.InvalidNtabCopyIdException +import com.twitter.frigate.pushservice.take.NtabCopyIdNotFoundException + +trait CandidateNTabCopy { + self: PushCandidate => + + def ntabCopy: MRNtabCopy = + ntabCopyId + .map(getNtabCopyFromCopyId).getOrElse( + throw new NtabCopyIdNotFoundException(s"NtabCopyId not found for $commonRecType")) + + private def getNtabCopyFromCopyId(ntabCopyId: Int): MRNtabCopy = + MrNtabCopyObjects + .getCopyFromId(ntabCopyId).getOrElse( + throw new InvalidNtabCopyIdException(s"Unknown NTab Copy ID: $ntabCopyId")) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.scala new file mode 100644 index 0000000000..4d6d67893f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/DiscoverTwitterNtabRequestHydrator.scala @@ -0,0 +1,58 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} +import com.twitter.notificationservice.thriftscala._ +import com.twitter.util.Future +import com.twitter.util.Time + +trait DiscoverTwitterNtabRequestHydrator extends NTabRequestHydrator { + self: PushCandidate => + + override val senderIdFut: Future[Long] = Future.value(0L) + + override val tapThroughFut: Future[String] = + commonRecType match { + case CRT.AddressBookUploadPush => Future.value(PushConstants.AddressBookUploadTapThrough) + case CRT.InterestPickerPush => Future.value(PushConstants.InterestPickerTapThrough) + case CRT.CompleteOnboardingPush => + Future.value(PushConstants.CompleteOnboardingInterestAddressTapThrough) + case _ => + Future.value(PushConstants.ConnectTabPushTapThrough) + } + + override val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = Future.Nil + + override val facepileUsersFut: Future[Seq[Long]] = Future.Nil + + override val storyContext: Option[StoryContext] = None + + override val inlineCard: Option[InlineCard] = None + + override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) + + override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = + if (self.commonRecType == CRT.ConnectTabPush || RecTypes.isOnboardingFlowType( + self.commonRecType)) { + Future.join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut).map { + case (senderId, displayTextEntities, facepileUsers, tapThrough) => + Some( + CreateGenericNotificationRequest( + userId = target.targetId, + senderId = senderId, + genericType = GenericType.RefreshableNotification, + displayText = DisplayText(values = displayTextEntities), + facepileUsers = facepileUsers, + timestampMillis = Time.now.inMillis, + tapThroughAction = Some(TapThroughAction(Some(tapThrough))), + impressionId = Some(impressionId), + socialProofText = socialProofDisplayText, + context = storyContext, + inlineCard = inlineCard, + refreshableType = refreshableType + )) + } + } else Future.None +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.scala new file mode 100644 index 0000000000..082bc17427 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/EventNTabRequestHydrator.scala @@ -0,0 +1,21 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.notificationservice.thriftscala.DisplayText +import com.twitter.notificationservice.thriftscala.InlineCard +import com.twitter.notificationservice.thriftscala.StoryContext +import com.twitter.util.Future + +trait EventNTabRequestHydrator extends NTabRequestHydrator { + self: PushCandidate => + + override def senderIdFut: Future[Long] = Future.value(0L) + + override def facepileUsersFut: Future[Seq[Long]] = Future.Nil + + override val storyContext: Option[StoryContext] = None + + override val inlineCard: Option[InlineCard] = None + + override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.scala new file mode 100644 index 0000000000..18662e2571 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/F1FirstDegreeTweetNTabRequestHydrator.scala @@ -0,0 +1,18 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.take.NotificationServiceSender +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.util.Future + +trait F1FirstDegreeTweetNTabRequestHydrator extends TweetNTabRequestHydrator { + self: PushCandidate with TweetCandidate with TweetAuthorDetails => + + override val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = + NotificationServiceSender.getDisplayTextEntityFromUser(tweetAuthor, "author", true).map(_.toSeq) + + override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.scala new file mode 100644 index 0000000000..8475256ad3 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ListCandidateNTabRequestHydrator.scala @@ -0,0 +1,34 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.pushservice.model.ListRecommendationPushCandidate +import com.twitter.notificationservice.thriftscala.DisplayText +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.notificationservice.thriftscala.InlineCard +import com.twitter.notificationservice.thriftscala.StoryContext +import com.twitter.notificationservice.thriftscala.TextValue +import com.twitter.util.Future + +trait ListCandidateNTabRequestHydrator extends NTabRequestHydrator { + + self: ListRecommendationPushCandidate => + + override lazy val senderIdFut: Future[Long] = + listOwnerId.map(_.getOrElse(0L)) + + override lazy val facepileUsersFut: Future[Seq[Long]] = Future.Nil + + override lazy val storyContext: Option[StoryContext] = None + + override lazy val inlineCard: Option[InlineCard] = None + + override lazy val tapThroughFut: Future[String] = Future.value(s"i/lists/${listId}") + + override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = listName.map { + listNameOpt => + listNameOpt.toSeq.map { name => + DisplayTextEntity(name = "title", value = TextValue.Text(name)) + } + } + + override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.scala new file mode 100644 index 0000000000..a245769a62 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutCreatorEventNtabRequestHydrator.scala @@ -0,0 +1,110 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutCreatorEventPushCandidate +import com.twitter.frigate.pushservice.take.NotificationServiceSender +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest +import com.twitter.notificationservice.thriftscala.DisplayText +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.notificationservice.thriftscala.GenericType +import com.twitter.notificationservice.thriftscala.InlineCard +import com.twitter.notificationservice.thriftscala.StoryContext +import com.twitter.notificationservice.thriftscala.TextValue +import com.twitter.notificationservice.thriftscala.TapThroughAction +import com.twitter.util.Future +import com.twitter.util.Time + +trait MagicFanoutCreatorEventNtabRequestHydrator extends NTabRequestHydrator { + self: PushCandidate with MagicFanoutCreatorEventPushCandidate => + + override val senderIdFut: Future[Long] = Future.value(creatorId) + + override lazy val tapThroughFut: Future[String] = + Future.value(s"/${userProfile.screenName}/superfollows/subscribe") + + lazy val optionalTweetCountEntityFut: Future[Option[DisplayTextEntity]] = { + creatorFanoutType match { + case CreatorFanoutType.UserSubscription => + numberOfTweetsFut.map { + _.flatMap { + case numberOfTweets if numberOfTweets >= 10 => + Some( + DisplayTextEntity( + name = "tweet_count", + emphasis = true, + value = TextValue.Text(numberOfTweets.toString))) + case _ => None + } + } + case _ => Future.None + } + } + + override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = + optionalTweetCountEntityFut + .map { tweetCountOpt => + Seq( + NotificationServiceSender + .getDisplayTextEntityFromUser(hydratedCreator, "display_name", isBold = true), + tweetCountOpt).flatten + } + + override lazy val facepileUsersFut: Future[Seq[Long]] = Future.value(Seq(creatorId)) + + override val storyContext: Option[StoryContext] = None + + override val inlineCard: Option[InlineCard] = None + + lazy val refreshableTypeFut = { + creatorFanoutType match { + case CreatorFanoutType.UserSubscription => + numberOfTweetsFut.map { + _.flatMap { + case numberOfTweets if numberOfTweets >= 10 => + Some("MagicFanoutCreatorSubscriptionWithTweets") + case _ => super.refreshableType + } + } + case _ => Future.value(super.refreshableType) + } + } + + override lazy val socialProofDisplayText: Option[DisplayText] = { + creatorFanoutType match { + case CreatorFanoutType.UserSubscription => + Some( + DisplayText(values = Seq( + DisplayTextEntity(name = "handle", value = TextValue.Text(userProfile.screenName))))) + case CreatorFanoutType.NewCreator => None + case _ => None + } + } + + override lazy val ntabRequest = { + Future + .join( + senderIdFut, + displayTextEntitiesFut, + facepileUsersFut, + tapThroughFut, + refreshableTypeFut).map { + case (senderId, displayTextEntities, facepileUsers, tapThrough, refreshableTypeOpt) => + Some( + CreateGenericNotificationRequest( + userId = target.targetId, + senderId = senderId, + genericType = GenericType.RefreshableNotification, + displayText = DisplayText(values = displayTextEntities), + facepileUsers = facepileUsers, + timestampMillis = Time.now.inMillis, + tapThroughAction = Some(TapThroughAction(Some(tapThrough))), + impressionId = Some(impressionId), + socialProofText = socialProofDisplayText, + context = storyContext, + inlineCard = inlineCard, + refreshableType = refreshableTypeOpt + )) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.scala new file mode 100644 index 0000000000..202533e3c1 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutNewsEventNTabRequestHydrator.scala @@ -0,0 +1,16 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.notificationservice.thriftscala.TextValue +import com.twitter.util.Future + +trait MagicFanoutNewsEventNTabRequestHydrator extends EventNTabRequestHydrator { + self: PushCandidate with MagicFanoutEventHydratedCandidate => + override lazy val tapThroughFut: Future[String] = Future.value(s"i/events/$eventId") + override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = + eventTitleFut.map { eventTitle => + Seq(DisplayTextEntity(name = "title", value = TextValue.Text(eventTitle))) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.scala new file mode 100644 index 0000000000..797dbe8907 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutProductLaunchNtabRequestHydrator.scala @@ -0,0 +1,97 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.notificationservice.thriftscala._ +import com.twitter.util.Future +import com.twitter.util.Time + +trait MagicFanoutProductLaunchNtabRequestHydrator extends NTabRequestHydrator { + self: PushCandidate with MagicFanoutProductLaunchCandidate => + + override val senderIdFut: Future[Long] = Future.value(0L) + + override lazy val tapThroughFut: Future[String] = Future.value(getProductLaunchTapThrough()) + + override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { + Future.value( + frigateNotification.magicFanoutProductLaunchNotification + .flatMap { + _.productInfo.flatMap { + _.body.map { body => + Seq( + DisplayTextEntity(name = "body", value = TextValue.Text(body)), + ) + } + } + }.getOrElse(Nil)) + } + + override lazy val facepileUsersFut: Future[Seq[Long]] = { + Future.value( + frigateNotification.magicFanoutProductLaunchNotification + .flatMap { + _.productInfo.flatMap { + _.facepileUsers + } + }.getOrElse(Nil)) + } + + override val storyContext: Option[StoryContext] = None + + override val inlineCard: Option[InlineCard] = None + + override lazy val socialProofDisplayText: Option[DisplayText] = { + frigateNotification.magicFanoutProductLaunchNotification.flatMap { + _.productInfo.flatMap { + _.title.map { title => + DisplayText(values = + Seq(DisplayTextEntity(name = "social_context", value = TextValue.Text(title)))) + } + } + } + } + + lazy val defaultTapThrough = target.params(PushFeatureSwitchParams.ProductLaunchTapThrough) + + private def getProductLaunchTapThrough(): String = { + frigateNotification.magicFanoutProductLaunchNotification match { + case Some(productLaunchNotif) => + productLaunchNotif.productInfo match { + case Some(productInfo) => productInfo.tapThrough.getOrElse(defaultTapThrough) + case _ => defaultTapThrough + } + case _ => defaultTapThrough + } + } + + private lazy val productLaunchNtabRequest: Future[Option[CreateGenericNotificationRequest]] = { + Future + .join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut) + .map { + case (senderId, displayTextEntities, facepileUsers, tapThrough) => + Some( + CreateGenericNotificationRequest( + userId = target.targetId, + senderId = senderId, + genericType = GenericType.RefreshableNotification, + displayText = DisplayText(values = displayTextEntities), + facepileUsers = facepileUsers, + timestampMillis = Time.now.inMillis, + tapThroughAction = Some(TapThroughAction(Some(tapThrough))), + impressionId = Some(impressionId), + socialProofText = socialProofDisplayText, + context = storyContext, + inlineCard = inlineCard, + refreshableType = refreshableType + )) + } + } + + override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { + if (target.params(PushFeatureSwitchParams.EnableNTabEntriesForProductLaunchNotifications)) { + productLaunchNtabRequest + } else Future.None + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.scala new file mode 100644 index 0000000000..ca3d9faf05 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/MagicFanoutSportsEventNTabRequestHydrator.scala @@ -0,0 +1,95 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate +import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest +import com.twitter.notificationservice.thriftscala.DisplayText +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.notificationservice.thriftscala.GenericType +import com.twitter.notificationservice.thriftscala.TextValue +import com.twitter.notificationservice.thriftscala.TapThroughAction +import com.twitter.util.Future +import com.twitter.util.Time + +trait MagicFanoutSportsEventNTabRequestHydrator extends EventNTabRequestHydrator { + self: PushCandidate + with MagicFanoutEventHydratedCandidate + with MagicFanoutSportsEventCandidate + with MagicFanoutSportsScoreInformation => + + lazy val stats = self.statsReceiver.scope("MagicFanoutSportsEventNtabHydrator") + lazy val inNetworkOnlyCounter = stats.counter("in_network_only") + lazy val facePilesEnabledCounter = stats.counter("face_piles_enabled") + lazy val facePilesDisabledCounter = stats.counter("face_piles_disabled") + lazy val filterPeopleWhoDontFollowMeCounter = stats.counter("pepole_who_dont_follow_me_counter") + + override lazy val tapThroughFut: Future[String] = { + Future.value(s"i/events/$eventId") + } + override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = + eventTitleFut.map { eventTitle => + Seq(DisplayTextEntity(name = "title", value = TextValue.Text(eventTitle))) + } + + override lazy val facepileUsersFut: Future[Seq[Long]] = + if (target.params(FS.EnableNTabFacePileForSportsEventNotifications)) { + Future + .join( + target.notificationsFromOnlyPeopleIFollow, + target.filterNotificationsFromPeopleThatDontFollowMe, + awayTeamInfo, + homeTeamInfo).map { + case (inNetworkOnly, filterPeopleWhoDontFollowMe, away, home) + if !(inNetworkOnly || filterPeopleWhoDontFollowMe) => + val awayTeamId = away.flatMap(_.twitterUserId) + val homeTeamId = home.flatMap(_.twitterUserId) + facePilesEnabledCounter.incr + Seq(awayTeamId, homeTeamId).flatten + case (inNetworkOnly, filterPeopleWhoDontFollowMe, _, _) => + facePilesDisabledCounter.incr + if (inNetworkOnly) inNetworkOnlyCounter.incr + if (filterPeopleWhoDontFollowMe) filterPeopleWhoDontFollowMeCounter.incr + Seq.empty[Long] + } + } else Future.Nil + + private lazy val sportsNtabRequest: Future[Option[CreateGenericNotificationRequest]] = { + Future + .join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut) + .map { + case (senderId, displayTextEntities, facepileUsers, tapThrough) => + Some( + CreateGenericNotificationRequest( + userId = target.targetId, + senderId = senderId, + genericType = GenericType.RefreshableNotification, + displayText = DisplayText(values = displayTextEntities), + facepileUsers = facepileUsers, + timestampMillis = Time.now.inMillis, + tapThroughAction = Some(TapThroughAction(Some(tapThrough))), + impressionId = Some(impressionId), + socialProofText = socialProofDisplayText, + context = storyContext, + inlineCard = inlineCard, + refreshableType = refreshableType + )) + } + } + + override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { + if (target.params(FS.EnableNTabEntriesForSportsEventNotifications)) { + self.target.history.flatMap { pushHistory => + val prevEventHistoryExists = pushHistory.sortedHistory.exists { + case (_, notification) => + notification.magicFanoutEventNotification.exists(_.eventId == self.eventId) + } + if (prevEventHistoryExists) { + Future.None + } else sportsNtabRequest + } + } else Future.None + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.scala new file mode 100644 index 0000000000..ea99ea68d4 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequest.scala @@ -0,0 +1,10 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest +import com.twitter.util.Future + +trait NTabRequest { + + def ntabRequest: Future[Option[CreateGenericNotificationRequest]] + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.scala new file mode 100644 index 0000000000..01df5365fa --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabRequestHydrator.scala @@ -0,0 +1,64 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationRequest +import com.twitter.notificationservice.thriftscala.DisplayText +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.notificationservice.thriftscala.GenericType +import com.twitter.notificationservice.thriftscala.InlineCard +import com.twitter.notificationservice.thriftscala.StoryContext +import com.twitter.notificationservice.thriftscala.TapThroughAction +import com.twitter.util.Future +import com.twitter.util.Time + +trait NTabRequestHydrator extends NTabRequest with CandidateNTabCopy { + self: PushCandidate => + + // Represents the sender of the recommendation + def senderIdFut: Future[Long] + + // Consists of a sequence representing the social context user ids. + def facepileUsersFut: Future[Seq[Long]] + + // Story Context is required for Tweet Recommendations + // Contains the Tweet ID of the recommended Tweet + def storyContext: Option[StoryContext] + + // Inline card used to render a generic notification. + def inlineCard: Option[InlineCard] + + // Represents where the recommendation should land when clicked + def tapThroughFut: Future[String] + + // Hydration for fields that are used within the NTab copy + def displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] + + // Represents the social proof text that is needed for specific NTab copies + def socialProofDisplayText: Option[DisplayText] + + // MagicRecs NTab entries always use RefreshableType as the Generic Type + final val genericType: GenericType = GenericType.RefreshableNotification + + def refreshableType: Option[String] = ntabCopy.refreshableType + + lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { + Future.join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut).map { + case (senderId, displayTextEntities, facepileUsers, tapThrough) => + Some( + CreateGenericNotificationRequest( + userId = target.targetId, + senderId = senderId, + genericType = GenericType.RefreshableNotification, + displayText = DisplayText(values = displayTextEntities), + facepileUsers = facepileUsers, + timestampMillis = Time.now.inMillis, + tapThroughAction = Some(TapThroughAction(Some(tapThrough))), + impressionId = Some(impressionId), + socialProofText = socialProofDisplayText, + context = storyContext, + inlineCard = inlineCard, + refreshableType = refreshableType + )) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.scala new file mode 100644 index 0000000000..17b43f457c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/NTabSocialContext.scala @@ -0,0 +1,46 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.base.SocialContextActions +import com.twitter.frigate.common.base.SocialContextUserDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.util.Future + +trait NTabSocialContext { + self: PushCandidate with SocialContextActions with SocialContextUserDetails => + + private def ntabDisplayUserIds: Seq[Long] = + socialContextUserIds.take(ntabDisplayUserIdsLength) + + def ntabDisplayUserIdsLength: Int = + if (socialContextUserIds.size == 2) 2 else 1 + + def ntabDisplayNamesAndIds: Future[Seq[(String, Long)]] = + scUserMap.map { userObjMap => + ntabDisplayUserIds.flatMap { id => + userObjMap(id).flatMap(_.profile.map(_.name)).map { name => (name, id) } + } + } + + def rankedNtabDisplayNamesAndIds(defaultToRecency: Boolean): Future[Seq[(String, Long)]] = + scUserMap.flatMap { userObjMap => + val rankedSocialContextActivityFut = + CandidateUtil.getRankedSocialContext( + socialContextActions, + target.seedsWithWeight, + defaultToRecency) + rankedSocialContextActivityFut.map { rankedSocialContextActivity => + val ntabDisplayUserIds = + rankedSocialContextActivity.map(_.userId).take(ntabDisplayUserIdsLength) + ntabDisplayUserIds.flatMap { id => + userObjMap(id).flatMap(_.profile.map(_.name)).map { name => (name, id) } + } + } + } + + def otherCount: Future[Int] = + ntabDisplayNamesAndIds.map { + case namesWithIdSeq => + Math.max(0, socialContextUserIds.length - namesWithIdSeq.size) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.scala new file mode 100644 index 0000000000..a2b99d1af8 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/OutOfNetworkTweetNTabRequestHydrator.scala @@ -0,0 +1,78 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.base.TopicCandidate +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.base.TweetDetails +import com.twitter.frigate.common.rec_types.RecTypes._ +import com.twitter.frigate.common.util.MrNtabCopyObjects +import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.take.NotificationServiceSender +import com.twitter.notificationservice.thriftscala.DisplayText +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.notificationservice.thriftscala.TextValue +import com.twitter.util.Future + +trait OutOfNetworkTweetNTabRequestHydrator extends TweetNTabRequestHydrator { + self: PushCandidate + with TweetCandidate + with TweetAuthorDetails + with TopicCandidate + with TweetDetails => + + lazy val useTopicCopyForMBCGNtab = mrModelingBasedTypes.contains(commonRecType) && target.params( + PushFeatureSwitchParams.EnableMrModelingBasedCandidatesTopicCopy) + lazy val useTopicCopyForFrsNtab = frsTypes.contains(commonRecType) && target.params( + PushFeatureSwitchParams.EnableFrsTweetCandidatesTopicCopy) + lazy val useTopicCopyForTagspaceNtab = tagspaceTypes.contains(commonRecType) && target.params( + PushFeatureSwitchParams.EnableHashspaceCandidatesTopicCopy) + + override lazy val tapThroughFut: Future[String] = { + if (hasVideo && self.target.params( + PushFeatureSwitchParams.EnableLaunchVideosInImmersiveExplore)) { + Future.value( + s"i/immersive_timeline?display_location=notification&include_pinned_tweet=true&pinned_tweet_id=${tweetId}&tl_type=imv") + } else { + tweetAuthor.map { + case Some(author) => + val authorProfile = author.profile.getOrElse( + throw new TweetNTabRequestHydratorException( + s"Unable to obtain author profile for: ${author.id}")) + s"${authorProfile.screenName}/status/${tweetId.toString}" + case _ => + throw new TweetNTabRequestHydratorException( + s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") + } + } + } + + override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = + if (localizedUttEntity.isDefined && + (useTopicCopyForMBCGNtab || useTopicCopyForFrsNtab || useTopicCopyForTagspaceNtab)) { + NotificationServiceSender + .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", isBold = true).map(_.toSeq) + } else { + NotificationServiceSender + .getDisplayTextEntityFromUser(tweetAuthor, "author", isBold = true).map(_.toSeq) + } + + override lazy val refreshableType: Option[String] = { + if (localizedUttEntity.isDefined && + (useTopicCopyForMBCGNtab || useTopicCopyForFrsNtab || useTopicCopyForTagspaceNtab)) { + MrNtabCopyObjects.TopicTweet.refreshableType + } else ntabCopy.refreshableType + } + + override def socialProofDisplayText: Option[DisplayText] = { + if (localizedUttEntity.isDefined && + (useTopicCopyForMBCGNtab || useTopicCopyForFrsNtab || useTopicCopyForTagspaceNtab)) { + localizedUttEntity.map(uttEntity => + DisplayText(values = + Seq(DisplayTextEntity("topic_name", TextValue.Text(uttEntity.localizedNameForDisplay))))) + } else None + } + + override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.scala new file mode 100644 index 0000000000..4673a001eb --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/ScheduledSpaceNTabRequestHydrator.scala @@ -0,0 +1,106 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.base.SpaceCandidate +import com.twitter.frigate.common.util.MrNtabCopyObjects +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.ScheduledSpaceSpeakerPushCandidate +import com.twitter.frigate.pushservice.model.ScheduledSpaceSubscriberPushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.take.NotificationServiceSender +import com.twitter.frigate.thriftscala.SpaceNotificationType +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.notificationservice.thriftscala._ +import com.twitter.util.Future +import com.twitter.util.Time + +trait ScheduledSpaceSpeakerNTabRequestHydrator extends ScheduledSpaceNTabRequestHydrator { + self: PushCandidate with ScheduledSpaceSpeakerPushCandidate => + + override def refreshableType: Option[String] = { + frigateNotification.spaceNotification.flatMap { spaceNotification => + spaceNotification.spaceNotificationType.flatMap { + case SpaceNotificationType.PreSpaceBroadcast => + MrNtabCopyObjects.ScheduledSpaceSpeakerSoon.refreshableType + case SpaceNotificationType.AtSpaceBroadcast => + MrNtabCopyObjects.ScheduledSpaceSpeakerNow.refreshableType + case _ => + throw new IllegalStateException(s"Unexpected SpaceNotificationType") + } + } + } + + override lazy val facepileUsersFut: Future[Seq[Long]] = Future.Nil + + override val socialProofDisplayText: Option[DisplayText] = Some(DisplayText()) +} + +trait ScheduledSpaceSubscriberNTabRequestHydrator extends ScheduledSpaceNTabRequestHydrator { + self: PushCandidate with ScheduledSpaceSubscriberPushCandidate => + + override lazy val facepileUsersFut: Future[Seq[Long]] = { + hostId match { + case Some(spaceHostId) => Future.value(Seq(spaceHostId)) + case _ => + Future.exception( + new IllegalStateException( + "Unable to get host id for ScheduledSpaceSubscriberNTabRequestHydrator")) + } + } + + override val socialProofDisplayText: Option[DisplayText] = None +} + +trait ScheduledSpaceNTabRequestHydrator extends NTabRequestHydrator { + self: PushCandidate with SpaceCandidate => + + def hydratedHost: Option[User] + + override lazy val senderIdFut: Future[Long] = { + hostId match { + case Some(spaceHostId) => Future.value(spaceHostId) + case _ => throw new IllegalStateException(s"No Space Host Id") + } + } + + override lazy val tapThroughFut: Future[String] = Future.value(s"i/spaces/$spaceId") + + override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = + NotificationServiceSender + .getDisplayTextEntityFromUser( + Future.value(hydratedHost), + fieldName = "space_host_name", + isBold = true + ).map(_.toSeq) + + override val storyContext: Option[StoryContext] = None + + override val inlineCard: Option[InlineCard] = None + + override lazy val ntabRequest: Future[Option[CreateGenericNotificationRequest]] = { + Future.join(senderIdFut, displayTextEntitiesFut, facepileUsersFut, tapThroughFut).map { + case (senderId, displayTextEntities, facepileUsers, tapThrough) => + val expiryTimeMillis = if (target.params(PushFeatureSwitchParams.EnableSpacesTtlForNtab)) { + Some( + (Time.now + target.params( + PushFeatureSwitchParams.SpaceNotificationsTTLDurationForNTab)).inMillis) + } else None + + Some( + CreateGenericNotificationRequest( + userId = target.targetId, + senderId = senderId, + genericType = GenericType.RefreshableNotification, + displayText = DisplayText(values = displayTextEntities), + facepileUsers = facepileUsers, + timestampMillis = Time.now.inMillis, + tapThroughAction = Some(TapThroughAction(Some(tapThrough))), + impressionId = Some(impressionId), + socialProofText = socialProofDisplayText, + context = storyContext, + inlineCard = inlineCard, + refreshableType = refreshableType, + expiryTimeMillis = expiryTimeMillis + )) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.scala new file mode 100644 index 0000000000..caa2a8cd09 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/SubscribedSearchTweetNtabRequestHydrator.scala @@ -0,0 +1,23 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.pushservice.model.SubscribedSearchTweetPushCandidate +import com.twitter.frigate.pushservice.take.NotificationServiceSender +import com.twitter.notificationservice.thriftscala.DisplayText +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.notificationservice.thriftscala.TextValue +import com.twitter.util.Future + +trait SubscribedSearchTweetNtabRequestHydrator extends TweetNTabRequestHydrator { + self: SubscribedSearchTweetPushCandidate => + override def displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = NotificationServiceSender + .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthor", isBold = true).map(_.toSeq) + + override def socialProofDisplayText: Option[DisplayText] = { + Some(DisplayText(values = Seq(DisplayTextEntity("search_query", TextValue.Text(searchTerm))))) + } + + override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) + + override lazy val tapThroughFut: Future[String] = + Future.value(self.ntabLandingUrl) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.scala new file mode 100644 index 0000000000..a67dee399c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopTweetImpressionsNTabRequestHydrator.scala @@ -0,0 +1,37 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.base.TopTweetImpressionsCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.notificationservice.thriftscala.DisplayText +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.notificationservice.thriftscala.InlineCard +import com.twitter.notificationservice.thriftscala.StoryContext +import com.twitter.notificationservice.thriftscala.StoryContextValue +import com.twitter.notificationservice.thriftscala.TextValue +import com.twitter.util.Future + +trait TopTweetImpressionsNTabRequestHydrator extends NTabRequestHydrator { + self: PushCandidate with TopTweetImpressionsCandidate => + + override lazy val tapThroughFut: Future[String] = + Future.value(s"${target.targetId}/status/$tweetId") + + override val senderIdFut: Future[Long] = Future.value(0L) + + override val facepileUsersFut: Future[Seq[Long]] = Future.Nil + + override val storyContext: Option[StoryContext] = + Some(StoryContext(altText = "", value = Some(StoryContextValue.Tweets(Seq(tweetId))))) + + override val inlineCard: Option[InlineCard] = None + + override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { + Future.value( + Seq( + DisplayTextEntity(name = "num_impressions", value = TextValue.Number(self.impressionsCount)) + ) + ) + } + + override def socialProofDisplayText: Option[DisplayText] = None +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.scala new file mode 100644 index 0000000000..17519efdaa --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TopicProofTweetNtabRequestHydrator.scala @@ -0,0 +1,60 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.pushservice.model.TopicProofTweetPushCandidate +import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException +import com.twitter.frigate.pushservice.exception.UttEntityNotFoundException +import com.twitter.frigate.pushservice.take.NotificationServiceSender +import com.twitter.notificationservice.thriftscala.DisplayText +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.notificationservice.thriftscala.StoryContext +import com.twitter.notificationservice.thriftscala.StoryContextValue +import com.twitter.notificationservice.thriftscala.TextValue +import com.twitter.util.Future + +trait TopicProofTweetNtabRequestHydrator extends NTabRequestHydrator { + self: TopicProofTweetPushCandidate => + + override def displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = NotificationServiceSender + .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", true) + .map(_.toSeq) + + private lazy val uttEntity = localizedUttEntity.getOrElse( + throw new UttEntityNotFoundException( + s"${getClass.getSimpleName} UttEntity missing for $tweetId") + ) + + override lazy val tapThroughFut: Future[String] = { + tweetAuthor.map { + case Some(author) => + val authorProfile = author.profile.getOrElse( + throw new TweetNTabRequestHydratorException( + s"Unable to obtain author profile for: ${author.id}")) + s"${authorProfile.screenName}/status/${tweetId.toString}" + case _ => + throw new TweetNTabRequestHydratorException( + s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") + } + } + + override lazy val socialProofDisplayText: Option[DisplayText] = { + Some( + DisplayText(values = + Seq(DisplayTextEntity("topic_name", TextValue.Text(uttEntity.localizedNameForDisplay)))) + ) + } + + override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) + + override val inlineCard = None + + override def storyContext: Option[StoryContext] = Some( + StoryContext("", Some(StoryContextValue.Tweets(Seq(tweetId))))) + + override def senderIdFut: Future[Long] = + tweetAuthor.map { + case Some(author) => author.id + case _ => + throw new TweetNTabRequestHydratorException( + s"Unable to obtain Author ID for: $commonRecType") + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.scala new file mode 100644 index 0000000000..07946a2202 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TrendTweetNtabHydrator.scala @@ -0,0 +1,61 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.base.TrendTweetCandidate +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.take.NotificationServiceSender +import com.twitter.frigate.pushservice.util.EmailLandingPageExperimentUtil +import com.twitter.notificationservice.thriftscala.DisplayText +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.notificationservice.thriftscala.TextValue +import com.twitter.util.Future + +trait TrendTweetNtabHydrator extends TweetNTabRequestHydrator { + self: PushCandidate with TrendTweetCandidate with TweetCandidate with TweetAuthorDetails => + + private lazy val trendTweetNtabStats = self.statsReceiver.scope("trend_tweet_ntab") + + private lazy val ruxLandingOnNtabCounter = + trendTweetNtabStats.counter("use_rux_landing_on_ntab") + + override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = + NotificationServiceSender + .getDisplayTextEntityFromUser(tweetAuthor, fieldName = "author_name", isBold = true) + .map( + _.toSeq :+ DisplayTextEntity( + name = "trend_name", + value = TextValue.Text(trendName), + emphasis = true) + ) + + override lazy val facepileUsersFut: Future[Seq[Long]] = senderIdFut.map(Seq(_)) + + override lazy val socialProofDisplayText: Option[DisplayText] = None + + override def refreshableType: Option[String] = ntabCopy.refreshableType + + override lazy val tapThroughFut: Future[String] = { + Future.join(tweetAuthor, target.deviceInfo).map { + case (Some(author), Some(deviceInfo)) => + val enableRuxLandingPage = deviceInfo.isRuxLandingPageEligible && target.params( + PushFeatureSwitchParams.EnableNTabRuxLandingPage) + val authorProfile = author.profile.getOrElse( + throw new TweetNTabRequestHydratorException( + s"Unable to obtain author profile for: ${author.id}")) + + if (enableRuxLandingPage) { + ruxLandingOnNtabCounter.incr() + EmailLandingPageExperimentUtil.createNTabRuxLandingURI(authorProfile.screenName, tweetId) + } else { + s"${authorProfile.screenName}/status/${tweetId.toString}" + } + + case _ => + throw new TweetNTabRequestHydratorException( + s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.scala new file mode 100644 index 0000000000..52a643b84c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetFavoriteNTabRequestHydrator.scala @@ -0,0 +1,38 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.base.SocialContextActions +import com.twitter.frigate.common.base.SocialContextUserDetails +import com.twitter.frigate.common.base.TweetAuthor +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.take.NotificationServiceSender +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.util.Future + +trait TweetFavoriteNTabRequestHydrator extends TweetNTabRequestHydrator with NTabSocialContext { + self: PushCandidate + with TweetCandidate + with TweetAuthor + with TweetAuthorDetails + with SocialContextActions + with SocialContextUserDetails => + + override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { + Future + .join( + NotificationServiceSender + .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", isBold = false), + NotificationServiceSender + .generateSocialContextTextEntities( + rankedNtabDisplayNamesAndIds(defaultToRecency = false), + otherCount) + ) + .map { + case (authorDisplay, socialContextDisplay) => + socialContextDisplay ++ authorDisplay + } + } + + override lazy val facepileUsersFut: Future[Seq[Long]] = Future.value(socialContextUserIds) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.scala new file mode 100644 index 0000000000..bfa8507f0b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetNTabRequestHydrator.scala @@ -0,0 +1,55 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.pushservice.exception.TweetNTabRequestHydratorException +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.notificationservice.thriftscala.InlineCard +import com.twitter.notificationservice.thriftscala.StoryContext +import com.twitter.notificationservice.thriftscala.StoryContextValue +import com.twitter.frigate.pushservice.util.EmailLandingPageExperimentUtil +import com.twitter.notificationservice.thriftscala._ +import com.twitter.util.Future + +trait TweetNTabRequestHydrator extends NTabRequestHydrator { + self: PushCandidate with TweetCandidate with TweetAuthorDetails => + + override def senderIdFut: Future[Long] = + tweetAuthor.map { + case Some(author) => author.id + case _ => + throw new TweetNTabRequestHydratorException( + s"Unable to obtain Author ID for: $commonRecType") + } + + override def storyContext: Option[StoryContext] = Some( + StoryContext( + altText = "", + value = Some(StoryContextValue.Tweets(Seq(tweetId))), + details = None + )) + + override def inlineCard: Option[InlineCard] = Some(InlineCard.TweetCard(TweetCard(tweetId))) + + override lazy val tapThroughFut: Future[String] = { + Future.join(tweetAuthor, target.deviceInfo).map { + case (Some(author), Some(deviceInfo)) => + val enableRuxLandingPage = deviceInfo.isRuxLandingPageEligible && target.params( + PushFeatureSwitchParams.EnableNTabRuxLandingPage) + val authorProfile = author.profile.getOrElse( + throw new TweetNTabRequestHydratorException( + s"Unable to obtain author profile for: ${author.id}")) + if (enableRuxLandingPage) { + EmailLandingPageExperimentUtil.createNTabRuxLandingURI(authorProfile.screenName, tweetId) + } else { + s"${authorProfile.screenName}/status/${tweetId.toString}" + } + case _ => + throw new TweetNTabRequestHydratorException( + s"Unable to obtain author and target details to generate tap through for Tweet: $tweetId") + } + } + + override def socialProofDisplayText: Option[DisplayText] = None +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.scala new file mode 100644 index 0000000000..c142fbfba2 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/model/ntab/TweetRetweetNTabRequestHydrator.scala @@ -0,0 +1,38 @@ +package com.twitter.frigate.pushservice.model.ntab + +import com.twitter.frigate.common.base.SocialContextActions +import com.twitter.frigate.common.base.SocialContextUserDetails +import com.twitter.frigate.common.base.TweetAuthor +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.take.NotificationServiceSender +import com.twitter.notificationservice.thriftscala.DisplayTextEntity +import com.twitter.util.Future + +trait TweetRetweetNTabRequestHydrator extends TweetNTabRequestHydrator with NTabSocialContext { + self: PushCandidate + with TweetCandidate + with TweetAuthor + with TweetAuthorDetails + with SocialContextActions + with SocialContextUserDetails => + + override lazy val displayTextEntitiesFut: Future[Seq[DisplayTextEntity]] = { + Future + .join( + NotificationServiceSender + .getDisplayTextEntityFromUser(tweetAuthor, "tweetAuthorName", isBold = false), + NotificationServiceSender + .generateSocialContextTextEntities( + rankedNtabDisplayNamesAndIds(defaultToRecency = false), + otherCount) + ) + .map { + case (authorDisplay, socialContextDisplay) => + socialContextDisplay ++ authorDisplay + } + } + + override lazy val facepileUsersFut: Future[Seq[Long]] = Future.value(socialContextUserIds) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.scala new file mode 100644 index 0000000000..238efe0bb1 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/DeployConfigModule.scala @@ -0,0 +1,68 @@ +package com.twitter.frigate.pushservice.module + +import com.google.inject.Provides +import com.google.inject.Singleton +import com.twitter.abdecider.LoggingABDecider +import com.twitter.decider.Decider +import com.twitter.featureswitches.v2.FeatureSwitches +import com.twitter.finagle.mtls.authentication.ServiceIdentifier +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.tunable.StandardTunableMap +import com.twitter.frigate.pushservice.config.DeployConfig +import com.twitter.frigate.pushservice.config.ProdConfig +import com.twitter.frigate.pushservice.config.StagingConfig +import com.twitter.frigate.pushservice.params.ShardParams +import com.twitter.inject.TwitterModule +import com.twitter.inject.annotations.Flag +import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ConfigRepoLocalPath +import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ServiceLocal + +object DeployConfigModule extends TwitterModule { + + @Provides + @Singleton + def providesDeployConfig( + @Flag(FlagName.numShards) numShards: Int, + @Flag(FlagName.shardId) shardId: Int, + @Flag(FlagName.isInMemCacheOff) inMemCacheOff: Boolean, + @Flag(ServiceLocal) isServiceLocal: Boolean, + @Flag(ConfigRepoLocalPath) localConfigRepoPath: String, + serviceIdentifier: ServiceIdentifier, + decider: Decider, + abDecider: LoggingABDecider, + featureSwitches: FeatureSwitches, + statsReceiver: StatsReceiver + ): DeployConfig = { + val tunableMap = if (serviceIdentifier.service.contains("canary")) { + StandardTunableMap(id = "frigate-pushservice-canary") + } else { StandardTunableMap(id = serviceIdentifier.service) } + val shardParams = ShardParams(numShards, shardId) + serviceIdentifier.environment match { + case "devel" | "staging" => + StagingConfig( + isServiceLocal = isServiceLocal, + localConfigRepoPath = localConfigRepoPath, + inMemCacheOff = inMemCacheOff, + decider = decider, + abDecider = abDecider, + featureSwitches = featureSwitches, + serviceIdentifier = serviceIdentifier, + tunableMap = tunableMap, + shardParams = shardParams + )(statsReceiver) + case "prod" => + ProdConfig( + isServiceLocal = isServiceLocal, + localConfigRepoPath = localConfigRepoPath, + inMemCacheOff = inMemCacheOff, + decider = decider, + abDecider = abDecider, + featureSwitches = featureSwitches, + serviceIdentifier = serviceIdentifier, + tunableMap = tunableMap, + shardParams = shardParams + )(statsReceiver) + case env => throw new Exception(s"Unknown environment $env") + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.scala new file mode 100644 index 0000000000..579f65acf1 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FilterModule.scala @@ -0,0 +1,16 @@ +package com.twitter.frigate.pushservice.module + +import com.google.inject.Provides +import javax.inject.Singleton +import com.twitter.discovery.common.nackwarmupfilter.NackWarmupFilter +import com.twitter.inject.TwitterModule +import com.twitter.inject.annotations.Flag +import com.twitter.util.Duration + +object FilterModule extends TwitterModule { + @Singleton + @Provides + def providesNackWarmupFilter( + @Flag(FlagName.nackWarmupDuration) warmupDuration: Duration + ): NackWarmupFilter = new NackWarmupFilter(warmupDuration) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.scala new file mode 100644 index 0000000000..4306e47cab --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/FlagModule.scala @@ -0,0 +1,56 @@ +package com.twitter.frigate.pushservice.module + +import com.twitter.app.Flag +import com.twitter.inject.TwitterModule +import com.twitter.util.Duration +import com.twitter.conversions.DurationOps._ + +object FlagName { + final val shardId = "service.shard" + final val numShards = "service.num_shards" + final val nackWarmupDuration = "service.nackWarmupDuration" + final val isInMemCacheOff = "service.isInMemCacheOff" +} + +object FlagModule extends TwitterModule { + + val shardId: Flag[Int] = flag[Int]( + name = FlagName.shardId, + help = "Service shard id" + ) + + val numShards: Flag[Int] = flag[Int]( + name = FlagName.numShards, + help = "Number of shards" + ) + + val mrLoggerIsTraceAll: Flag[Boolean] = flag[Boolean]( + name = "service.isTraceAll", + help = "atraceflag", + default = false + ) + + val mrLoggerNthLog: Flag[Boolean] = flag[Boolean]( + name = "service.nthLog", + help = "nthlog", + default = false + ) + + val inMemCacheOff: Flag[Boolean] = flag[Boolean]( + name = FlagName.isInMemCacheOff, + help = "is inMemCache Off (currently only applies for user_health_model_score_store_cache)", + default = false + ) + + val mrLoggerNthVal: Flag[Long] = flag[Long]( + name = "service.nthVal", + help = "nthlogval", + default = 0, + ) + + val nackWarmupDuration: Flag[Duration] = flag[Duration]( + name = FlagName.nackWarmupDuration, + help = "duration to nack at startup", + default = 0.seconds + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.scala new file mode 100644 index 0000000000..d4bceb549a --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/LoggedOutPushTargetUserBuilderModule.scala @@ -0,0 +1,27 @@ +package com.twitter.frigate.pushservice.module + +import com.google.inject.Provides +import com.google.inject.Singleton +import com.twitter.decider.Decider +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.target.LoggedOutPushTargetUserBuilder +import com.twitter.frigate.pushservice.config.DeployConfig +import com.twitter.inject.TwitterModule + +object LoggedOutPushTargetUserBuilderModule extends TwitterModule { + + @Provides + @Singleton + def providesLoggedOutPushTargetUserBuilder( + decider: Decider, + config: DeployConfig, + statsReceiver: StatsReceiver + ): LoggedOutPushTargetUserBuilder = { + LoggedOutPushTargetUserBuilder( + historyStore = config.loggedOutHistoryStore, + inputDecider = decider, + inputAbDecider = config.abDecider, + loggedOutPushInfoStore = config.loggedOutPushInfoStore + )(statsReceiver) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.scala new file mode 100644 index 0000000000..c71ff24ddb --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushHandlerModule.scala @@ -0,0 +1,78 @@ +package com.twitter.frigate.pushservice.module + +import com.google.inject.Provides +import com.google.inject.Singleton +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.target.LoggedOutPushTargetUserBuilder +import com.twitter.frigate.pushservice.refresh_handler.RefreshForPushHandler +import com.twitter.frigate.pushservice.config.DeployConfig +import com.twitter.frigate.pushservice.send_handler.SendHandler +import com.twitter.frigate.pushservice.take.candidate_validator.RFPHCandidateValidator +import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPostCandidateValidator +import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPreCandidateValidator +import com.twitter.frigate.pushservice.refresh_handler.LoggedOutRefreshForPushHandler +import com.twitter.frigate.pushservice.take.SendHandlerNotifier +import com.twitter.frigate.pushservice.target.PushTargetUserBuilder +import com.twitter.inject.TwitterModule + +object PushHandlerModule extends TwitterModule { + + @Provides + @Singleton + def providesRefreshForPushHandler( + pushTargetUserBuilder: PushTargetUserBuilder, + config: DeployConfig, + statsReceiver: StatsReceiver + ): RefreshForPushHandler = { + new RefreshForPushHandler( + pushTargetUserBuilder = pushTargetUserBuilder, + candSourceGenerator = config.candidateSourceGenerator, + rfphRanker = config.rfphRanker, + candidateHydrator = config.candidateHydrator, + candidateValidator = new RFPHCandidateValidator(config), + rfphTakeStepUtil = config.rfphTakeStepUtil, + rfphRestrictStep = config.rfphRestrictStep, + rfphNotifier = config.rfphNotifier, + rfphStatsRecorder = config.rfphStatsRecorder, + mrRequestScriberNode = config.mrRequestScriberNode, + rfphFeatureHydrator = config.rfphFeatureHydrator, + rfphPrerankFilter = config.rfphPrerankFilter, + rfphLightRanker = config.rfphLightRanker + )(statsReceiver) + } + + @Provides + @Singleton + def providesSendHandler( + pushTargetUserBuilder: PushTargetUserBuilder, + config: DeployConfig, + statsReceiver: StatsReceiver + ): SendHandler = { + new SendHandler( + pushTargetUserBuilder, + new SendHandlerPreCandidateValidator(config), + new SendHandlerPostCandidateValidator(config), + new SendHandlerNotifier(config.candidateNotifier, statsReceiver.scope("SendHandlerNotifier")), + config.sendHandlerCandidateHydrator, + config.featureHydrator, + config.sendHandlerPredicateUtil, + config.mrRequestScriberNode)(statsReceiver, config) + } + + @Provides + @Singleton + def providesLoggedOutRefreshForPushHandler( + loPushTargetUserBuilder: LoggedOutPushTargetUserBuilder, + config: DeployConfig, + statsReceiver: StatsReceiver + ): LoggedOutRefreshForPushHandler = { + new LoggedOutRefreshForPushHandler( + loPushTargetUserBuilder = loPushTargetUserBuilder, + loPushCandidateSourceGenerator = config.loCandidateSourceGenerator, + candidateHydrator = config.candidateHydrator, + loRanker = config.loggedOutRFPHRanker, + loRfphNotifier = config.loRfphNotifier, + loMrRequestScriberNode = config.loggedOutMrRequestScriberNode, + )(statsReceiver) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.scala new file mode 100644 index 0000000000..97e4844924 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushServiceDarkTrafficModule.scala @@ -0,0 +1,33 @@ +package com.twitter.frigate.pushservice.module + +import com.google.inject.Singleton +import com.twitter.decider.Decider +import com.twitter.decider.RandomRecipient +import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient +import com.twitter.frigate.pushservice.thriftscala.PushService +import com.twitter.inject.Injector +import com.twitter.inject.thrift.modules.ReqRepDarkTrafficFilterModule + +/** + * The darkTraffic filter sample all requests by default + and set the diffy dest to nil for non prod environments + */ +@Singleton +object PushServiceDarkTrafficModule + extends ReqRepDarkTrafficFilterModule[PushService.ReqRepServicePerEndpoint] + with MtlsClient { + + override def label: String = "frigate-pushservice-diffy-proxy" + + /** + * Function to determine if the request should be "sampled", e.g. + * sent to the dark service. + * + * @param injector the [[com.twitter.inject.Injector]] for use in determining if a given request + * should be forwarded or not. + */ + override protected def enableSampling(injector: Injector): Any => Boolean = { + val decider = injector.instance[Decider] + _ => decider.isAvailable("frigate_pushservice_dark_traffic_percent", Some(RandomRecipient)) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.scala new file mode 100644 index 0000000000..ccdd1f1108 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/PushTargetUserBuilderModule.scala @@ -0,0 +1,64 @@ +package com.twitter.frigate.pushservice.module + +import com.google.inject.Provides +import com.google.inject.Singleton +import com.twitter.decider.Decider +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.config.DeployConfig +import com.twitter.frigate.pushservice.target.PushTargetUserBuilder +import com.twitter.inject.TwitterModule + +object PushTargetUserBuilderModule extends TwitterModule { + + @Provides + @Singleton + def providesPushTargetUserBuilder( + decider: Decider, + config: DeployConfig, + statsReceiver: StatsReceiver + ): PushTargetUserBuilder = { + PushTargetUserBuilder( + historyStore = config.historyStore, + emailHistoryStore = config.emailHistoryStore, + labeledPushRecsStore = config.labeledPushRecsDecideredStore, + onlineUserHistoryStore = config.onlineUserHistoryStore, + pushRecItemsStore = config.pushRecItemStore, + userStore = config.safeUserStore, + pushInfoStore = config.pushInfoStore, + userCountryStore = config.userCountryStore, + userUtcOffsetStore = config.userUtcOffsetStore, + dauProbabilityStore = config.dauProbabilityStore, + nsfwConsumerStore = config.nsfwConsumerStore, + genericNotificationFeedbackStore = config.genericNotificationFeedbackStore, + userFeatureStore = config.userFeaturesStore, + mrUserStateStore = config.mrUserStatePredictionStore, + tweetImpressionStore = config.tweetImpressionStore, + timelinesUserSessionStore = config.timelinesUserSessionStore, + cachedTweetyPieStore = config.cachedTweetyPieStoreV2, + strongTiesStore = config.strongTiesStore, + userHTLLastVisitStore = config.userHTLLastVisitStore, + userLanguagesStore = config.userLanguagesStore, + inputDecider = decider, + inputAbDecider = config.abDecider, + realGraphScoresTop500InStore = config.realGraphScoresTop500InStore, + recentFollowsStore = config.recentFollowsStore, + resurrectedUserStore = config.reactivatedUserInfoStore, + configParamsBuilder = config.configParamsBuilder, + optOutUserInterestsStore = config.optOutUserInterestsStore, + deviceInfoStore = config.deviceInfoStore, + pushcapDynamicPredictionStore = config.pushcapDynamicPredictionStore, + appPermissionStore = config.appPermissionStore, + optoutModelScorer = config.optoutModelScorer, + userTargetingPropertyStore = config.userTargetingPropertyStore, + ntabCaretFeedbackStore = config.ntabCaretFeedbackStore, + genericFeedbackStore = config.genericFeedbackStore, + inlineActionHistoryStore = config.inlineActionHistoryStore, + featureHydrator = config.featureHydrator, + openAppUserStore = config.openAppUserStore, + openedPushByHourAggregatedStore = config.openedPushByHourAggregatedStore, + geoduckStoreV2 = config.geoDuckV2Store, + superFollowEligibilityUserStore = config.superFollowEligibilityUserStore, + superFollowApplicationStatusStore = config.superFollowApplicationStatusStore + )(statsReceiver) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.scala new file mode 100644 index 0000000000..049d731a31 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/module/ThriftWebFormsModule.scala @@ -0,0 +1,9 @@ +package com.twitter.frigate.pushservice.module + +import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule +import com.twitter.finatra.thrift.ThriftServer +import com.twitter.frigate.pushservice.thriftscala.PushService + +class ThriftWebFormsModule(server: ThriftServer) + extends MtlsThriftWebFormsModule[PushService.MethodPerEndpoint](server) { +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.scala new file mode 100644 index 0000000000..9c17ea5f24 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/DeciderKey.scala @@ -0,0 +1,210 @@ +package com.twitter.frigate.pushservice.params + +import com.twitter.servo.decider.DeciderKeyEnum + +object DeciderKey extends DeciderKeyEnum { + val disableAllRelevance = Value("frigate_pushservice_disable_all_relevance") + val disableHeavyRanking = Value("frigate_pushservice_disable_heavy_ranking") + val restrictLightRanking = Value("frigate_pushservice_restrict_light_ranking") + val downSampleLightRankingScribeCandidates = Value( + "frigate_pushservice_down_sample_light_ranking_scribe_candidates") + val entityGraphTweetRecsDeciderKey = Value("user_tweet_entity_graph_tweet_recs") + val enablePushserviceWritesToNotificationServiceDeciderKey = Value( + "frigate_pushservice_enable_writes_to_notification_service") + val enablePushserviceWritesToNotificationServiceForAllEmployeesDeciderKey = Value( + "frigate_pushservice_enable_writes_to_notification_service_for_employees") + val enablePushserviceWritesToNotificationServiceForEveryoneDeciderKey = Value( + "frigate_pushservice_enable_writes_to_notification_service_for_everyone") + val enablePromptFeedbackFatigueResponseNoPredicateDeciderKey = Value( + "frigate_pushservice_enable_ntab_feedback_prompt_response_no_filter_predicate") + val enablePushserviceDeepbirdv2CanaryClusterDeciderKey = Value( + "frigate_pushservice_canary_enable_deepbirdv2_canary_cluster") + val enableUTEGSCForEarlybirdTweetsDecider = Value( + "frigate_pushservice_enable_uteg_sc_for_eb_tweets") + val enableTweetFavRecs = Value("frigate_pushservice_enable_tweet_fav_recs") + val enableTweetRetweetRecs = Value("frigate_pushservice_enable_tweet_retweet_recs") + val enablePushSendEventBus = Value("frigate_pushservice_enable_push_send_eventbus") + + val enableModelBasedPushcapAssignments = Value( + "frigate_pushservice_enable_model_based_pushcap_assignments") + + val enableTweetAnnotationFeatureHydration = Value( + "frigate_pushservice_enable_tweet_annotation_features_hydration") + val enableMrRequestScribing = Value("frigate_pushservice_enable_mr_request_scribing") + val enableHighQualityCandidateScoresScribing = Value( + "frigate_pushservice_enable_high_quality_candidate_scribing") + val enableHtlUserAuthorRealTimeAggregateFeatureHydration = Value( + "frigate_pushservice_enable_htl_new_user_user_author_rta_hydration") + val enableMrUserSemanticCoreFeaturesHydration = Value( + "frigate_pushservice_enable_mr_user_semantic_core_feature_hydration") + val enableMrUserSemanticCoreNoZeroFeaturesHydration = Value( + "frigate_pushservice_enable_mr_user_semantic_core_no_zero_feature_hydration") + val enableHtlOfflineUserAggregateExtendedFeaturesHydration = Value( + "frigate_pushservice_enable_htl_offline_user_aggregate_extended_features_hydration") + val enableNerErgFeaturesHydration = Value("frigate_pushservice_enable_ner_erg_features_hydration") + val enableDaysSinceRecentResurrectionFeatureHydration = Value( + "frigate_pushservice_enable_days_since_recent_resurrection_features_hydration") + val enableUserPastAggregatesFeatureHydration = Value( + "frigate_pushservice_enable_user_past_aggregates_features_hydration") + val enableUserSignalLanguageFeatureHydration = Value( + "frigate_pushservice_enable_user_signal_language_features_hydration") + val enableUserPreferredLanguageFeatureHydration = Value( + "frigate_pushservice_enable_user_preferred_language_features_hydration") + val enablePredicateDetailedInfoScribing = Value( + "frigate_pushservice_enable_predicate_detailed_info_scribing") + val enablePushCapInfoScribing = Value("frigate_pushservice_enable_push_cap_info_scribing") + val disableMLInFiltering = Value("frigate_pushservice_disable_ml_in_filtering") + val useHydratedLabeledSendsForFeaturesDeciderKey = Value( + "use_hydrated_labeled_sends_for_features") + val verifyHydratedLabeledSendsForFeaturesDeciderKey = Value( + "verify_hydrated_labeled_sends_for_features") + val trainingDataDeciderKey = Value("frigate_notifier_quality_model_training_data") + val skipMlModelPredicateDeciderKey = Value("skip_ml_model_predicate") + val scribeModelFeaturesDeciderKey = Value("scribe_model_features") + val scribeModelFeaturesWithoutHydratingNewFeaturesDeciderKey = Value( + "scribe_model_features_without_hydrating_new_features") + val scribeModelFeaturesForRequestScribe = Value("scribe_model_features_for_request_scribe") + val enableMrUserSimclusterV2020FeaturesHydration = Value( + "frigate_pushservice_enable_mr_user_simcluster_v2020_hydration") + val enableMrUserSimclusterV2020NoZeroFeaturesHydration = Value( + "frigate_pushservice_enable_mr_user_simcluster_v2020_no_zero_feature_hydration") + val enableMrUserEngagedTweetTokensFeaturesHydration = Value( + "frigate_pushservice_enable_mr_user_engaged_tweet_tokens_feature_hydration") + val enableMrCandidateTweetTokensFeaturesHydration = Value( + "frigate_pushservice_enable_mr_candidate_tweet_tokens_feature_hydration") + val enableTopicEngagementRealTimeAggregatesFeatureHydration = Value( + "frigate_pushservice_enable_topic_engagement_real_time_aggregates_feature_hydration" + ) + val enableUserTopicAggregatesFeatureHydration = Value( + "frigate_pushservice_enable_user_topic_aggregates_feature_hydration" + ) + val enableDurationSinceLastVisitFeatureHydration = Value( + "frigate_pushservice_enable_duration_since_last_visit_features_hydration" + ) + val enableTwistlyAggregatesFeatureHydration = Value( + "frigate_pushservice_enable_twistly_agg_feature_hydration" + ) + val enableTwHINUserEngagementFeaturesHydration = Value( + "frigate_pushservice_enable_twhin_user_engagement_features_hydration" + ) + val enableTwHINUserFollowFeaturesHydration = Value( + "frigate_pushservice_enable_twhin_user_follow_features_hydration" + ) + val enableTwHINAuthorFollowFeaturesHydration = Value( + "frigate_pushservice_enable_twhin_author_follow_features_hydration" + ) + val enableTweetTwHINFavFeaturesHydration = Value( + "frigate_pushservice_enable_tweet_twhin_fav_features_hydration" + ) + val enableSpaceVisibilityLibraryFiltering = Value( + "frigate_pushservice_enable_space_visibility_library_filtering" + ) + val enableVfFeatureHydrationSpaceShim = Value( + "frigate_pushservice_enable_visibility_filtering_feature_hydration_in_space_shim") + val enableUserTopicFollowFeatureSet = Value( + "frigate_pushservice_enable_user_topic_follow_feature_hydration") + val enableOnboardingNewUserFeatureSet = Value( + "frigate_pushservice_enable_onboarding_new_user_feature_hydration") + val enableMrUserTopicSparseContFeatureSet = Value( + "frigate_pushservice_enable_mr_user_topic_sparse_cont_feature_hydration" + ) + val enableUserPenguinLanguageFeatureSet = Value( + "frigate_pushservice_enable_user_penguin_language_feature_hydration") + val enableMrUserHashspaceEmbeddingFeatureSet = Value( + "frigate_pushservice_enable_mr_user_hashspace_embedding_feature_hydration") + val enableMrUserAuthorSparseContFeatureSet = Value( + "frigate_pushservice_enable_mr_user_author_sparse_cont_feature_hydration" + ) + val enableMrTweetSentimentFeatureSet = Value( + "frigate_pushservice_enable_mr_tweet_sentiment_feature_hydration" + ) + val enableMrTweetAuthorAggregatesFeatureSet = Value( + "frigate_pushservice_enable_mr_tweet_author_aggregates_feature_hydration" + ) + val enableUserGeoFeatureSet = Value("frigate_pushservice_enable_user_geo_feature_hydration") + val enableAuthorGeoFeatureSet = Value("frigate_pushservice_enable_author_geo_feature_hydration") + + val rampupUserGeoFeatureSet = Value("frigate_pushservice_ramp_up_user_geo_feature_hydration") + val rampupAuthorGeoFeatureSet = Value("frigate_pushservice_ramp_up_author_geo_feature_hydration") + + val enablePopGeoTweets = Value("frigate_pushservice_enable_pop_geo_tweets") + val enableTrendsTweets = Value("frigate_pushservice_enable_trends_tweets") + val enableTripGeoTweetCandidates = Value("frigate_pushservice_enable_trip_geo_tweets") + val enableContentRecommenderMixerAdaptor = Value( + "frigate_pushservice_enable_content_recommender_mixer_adaptor") + val enableGenericCandidateAdaptor = Value("frigate_pushservice_enable_generic_candidate_adaptor") + val enableTripGeoTweetContentMixerDarkTraffic = Value( + "frigate_pushservice_enable_trip_geo_tweets_content_mixer_dark_traffic") + + val enableInsTraffic = Value("frigate_pushservice_enable_ins_traffic") + val enableIsTweetTranslatable = Value("frigate_pushservice_enable_is_tweet_translatable") + + val enableMrTweetSimClusterFeatureSet = Value( + "frigate_pushservice_enable_mr_tweet_simcluster_feature_hydration") + + val enableMrOfflineUserTweetTopicAggregate = Value( + "frigate_pushservice_enable_mr_offline_user_tweet_topic_aggregate_hydration") + + val enableMrOfflineUserTweetSimClusterAggregate = Value( + "frigate_pushservice_enable_mr_offline_user_tweet_simcluster_aggregate_hydration" + ) + val enableRealGraphV2FeatureHydration = Value( + "frigate_pushservice_enable_real_graph_v2_features_hydration") + + val enableTweetBeTFeatureHydration = Value( + "frigate_pushservice_enable_tweet_bet_features_hydration") + + val enableInvalidatingCachedHistoryStoreAfterWrites = Value( + "frigate_pushservice_enable_invalidating_cached_history_store_after_writes") + + val enableInvalidatingCachedLoggedOutHistoryStoreAfterWrites = Value( + "frigate_pushservice_enable_invalidating_cached_logged_out_history_store_after_writes") + + val enableUserSendTimeFeatureHydration = Value( + "frigate_pushservice_enable_user_send_time_feature_hydration" + ) + + val enablePnegMultimodalPredictionForF1Tweets = Value( + "frigate_pushservice_enable_pneg_multimodal_prediction_for_f1_tweets" + ) + + val enableScribingOonFavScoreForF1Tweets = Value( + "frigate_pushservice_enable_oon_fav_scribe_for_f1_tweets" + ) + + val enableMrUserUtcSendTimeAggregateFeaturesHydration = Value( + "frigate_pushservice_enable_mr_user_utc_send_time_aggregate_hydration" + ) + + val enableMrUserLocalSendTimeAggregateFeaturesHydration = Value( + "frigate_pushservice_enable_mr_user_local_send_time_aggregate_hydration" + ) + + val enableBqmlReportModelPredictionForF1Tweets = Value( + "frigate_pushservice_enable_bqml_report_model_prediction_for_f1_tweets" + ) + + val enableUserTwhinEmbeddingFeatureHydration = Value( + "frigate_pushservice_enable_user_twhin_embedding_feature_hydration" + ) + + val enableAuthorFollowTwhinEmbeddingFeatureHydration = Value( + "frigate_pushservice_enable_author_follow_twhin_embedding_feature_hydration" + ) + + val enableScribingMLFeaturesAsDataRecord = Value( + "frigate_pushservice_enable_scribing_ml_features_as_datarecord" + ) + + val enableDirectHydrationForUserFeatures = Value( + "frigate_pushservice_enable_direct_hydration_for_user_features" + ) + + val enableAuthorVerifiedFeatureHydration = Value( + "frigate_pushservice_enable_author_verified_feature_hydration" + ) + + val enableAuthorCreatorSubscriptionFeatureHydration = Value( + "frigate_pushservice_enable_author_creator_subscription_feature_hydration" + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.scala new file mode 100644 index 0000000000..cd96b49348 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushConstants.scala @@ -0,0 +1,126 @@ +package com.twitter.frigate.pushservice.params + +import com.twitter.conversions.DurationOps._ +import com.twitter.frigate.user_states.thriftscala.UserState +import java.util.Locale + +object PushConstants { + + final val ServiceProdEnvironmentName = "prod" + + final val RestrictLightRankingCandidatesThreshold = 1 + + final val DownSampleLightRankingScribeCandidatesRate = 1 + + final val NewUserLookbackWindow = 1.days + + final val PushCapInactiveUserAndroid = 1 + final val PushCapInactiveUserIos = 1 + final val PushCapLightOccasionalOpenerUserAndroid = 1 + final val PushCapLightOccasionalOpenerUserIos = 1 + + final val UserStateToPushCapIos = Map( + UserState.Inactive.name -> PushCapInactiveUserIos, + UserState.LightOccasionalOpener.name -> PushCapLightOccasionalOpenerUserIos + ) + final val UserStateToPushCapAndroid = Map( + UserState.Inactive.name -> PushCapInactiveUserAndroid, + UserState.LightOccasionalOpener.name -> PushCapLightOccasionalOpenerUserAndroid + ) + + final val AcceptableTimeSinceLastNegativeResponse = 1.days + + final val DefaultLookBackForHistory = 1.hours + + final val DefaultEventMediaUrl = "" + + final val ConnectTabPushTapThrough = "i/connect_people" + + final val AddressBookUploadTapThrough = "i/flow/mr-address-book-upload" + final val InterestPickerTapThrough = "i/flow/mr-interest-picker" + final val CompleteOnboardingInterestAddressTapThrough = "i/flow/mr-interest-address" + + final val IndiaCountryCode = "IN" + final val JapanCountryCode = Locale.JAPAN.getCountry.toUpperCase + final val UKCountryCode = Locale.UK.getCountry.toUpperCase + + final val IndiaTimeZoneCode = "Asia/Kolkata" + final val JapanTimeZoneCode = "Asia/Tokyo" + final val UKTimeZoneCode = "Europe/London" + + final val countryCodeToTimeZoneMap = Map( + IndiaCountryCode -> IndiaTimeZoneCode, + JapanCountryCode -> JapanTimeZoneCode, + UKCountryCode -> UKTimeZoneCode + ) + + final val AbuseStrike_Top2Percent_Id = "AbuseStrike_Top2Percent_Id" + final val AbuseStrike_Top1Percent_Id = "AbuseStrike_Top1Percent_Id" + final val AbuseStrike_Top05Percent_Id = "AbuseStrike_Top05Percent_Id" + final val AbuseStrike_Top025Percent_Id = "AbuseStrike_Top025Percent_Id" + final val AllSpamReportsPerFav_Top1Percent_Id = "AllSpamReportsPerFav_Top1Percent_Id" + final val ReportsPerFav_Top1Percent_Id = "ReportsPerFav_Top1Percent_Id" + final val ReportsPerFav_Top2Percent_Id = "ReportsPerFav_Top2Percent_Id" + final val MediaUnderstanding_Nudity_Id = "MediaUnderstanding_Nudity_Id" + final val MediaUnderstanding_Beauty_Id = "MediaUnderstanding_Beauty_Id" + final val MediaUnderstanding_SinglePerson_Id = "MediaUnderstanding_SinglePerson_Id" + final val PornList_Id = "PornList_Id" + final val PornographyAndNsfwContent_Id = "PornographyAndNsfwContent_Id" + final val SexLife_Id = "SexLife_Id" + final val SexLifeOrSexualOrientation_Id = "SexLifeOrSexualOrientation_Id" + final val ProfanityFilter_Id = "ProfanityFilter_Id" + final val TweetSemanticCoreIdFeature = "tweet.core.tweet.semantic_core_annotations" + final val targetUserGenderFeatureName = "Target.User.Gender" + final val targetUserAgeFeatureName = "Target.User.AgeBucket" + final val targetUserPreferredLanguage = "user.language.user.preferred_contents" + final val tweetAgeInHoursFeatureName = "RecTweet.TweetyPieResult.TweetAgeInHrs" + final val authorActiveFollowerFeatureName = "RecTweetAuthor.User.ActiveFollowers" + final val favFeatureName = "tweet.core.tweet_counts.favorite_count" + final val sentFeatureName = + "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_sent.any_feature.Duration.Top.count" + final val authorSendCountFeatureName = + "tweet_author_aggregate.pair.any_label.any_feature.28.days.count" + final val authorReportCountFeatureName = + "tweet_author_aggregate.pair.label.reportTweetDone.any_feature.28.days.count" + final val authorDislikeCountFeatureName = + "tweet_author_aggregate.pair.label.ntab.isDisliked.any_feature.28.days.count" + final val TweetLikesFeatureName = "tweet.core.tweet_counts.favorite_count" + final val TweetRepliesFeatureName = "tweet.core.tweet_counts.reply_count" + + final val EnableCopyFeaturesForIbis2ModelValues = "has_copy_features" + + final val EmojiFeatureNameForIbis2ModelValues = "emoji" + + final val TargetFeatureNameForIbis2ModelValues = "target" + + final val CopyBodyExpIbisModelValues = "enable_body_exp" + + final val TweetMediaEmbeddingBQKeyIds = Seq( + 230, 110, 231, 111, 232, 233, 112, 113, 234, 235, 114, 236, 115, 237, 116, 117, 238, 118, 239, + 119, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 240, 120, 241, 121, 242, 0, 1, 122, 243, 244, 123, + 2, 124, 245, 3, 4, 246, 125, 5, 126, 247, 127, 248, 6, 128, 249, 7, 8, 129, 9, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 250, 130, 251, 252, 131, 132, 253, 133, 254, 134, 255, 135, 136, 137, + 138, 139, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 140, 141, 142, 143, 144, 145, 146, 147, 148, + 149, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 60, + 61, 62, 63, 64, 65, 66, 67, 68, 69, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 90, 91, 92, 93, + 94, 95, 96, 97, 98, 99, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, 217, 218, 219, 220, 100, 221, 101, 222, 223, 102, 224, 103, 104, 225, 105, 226, + 227, 106, 107, 228, 108, 229, 109 + ) + + final val SportsEventDomainId = 6L + + final val OoncQualityCombinedScore = "OoncQualityCombinedScore" +} + +object PushQPSLimitConstants { + + final val PerspectiveStoreQPS = 100000 + + final val IbisOrNTabQPSForRFPH = 100000 + + final val SocialGraphServiceBatchSize = 100000 +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.scala new file mode 100644 index 0000000000..370e400767 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushEnums.scala @@ -0,0 +1,135 @@ +package com.twitter.frigate.pushservice.params + +/** + * Enum for naming scores we will scribe for non-personalized high quality candidate generation + */ +object HighQualityScribingScores extends Enumeration { + type Name = Value + val HeavyRankingScore = Value + val NonPersonalizedQualityScoreUsingCnn = Value + val BqmlNsfwScore = Value + val BqmlReportScore = Value +} + +/** + * Enum for quality upranking transform + */ +object MrQualityUprankingTransformTypeEnum extends Enumeration { + val Linear = Value + val Sigmoid = Value +} + +/** + * Enum for quality partial upranking transform + */ +object MrQualityUprankingPartialTypeEnum extends Enumeration { + val All = Value + val Oon = Value +} + +/** + * Enum for bucket membership in DDG 10220 Mr Bold Title Favorite Retweet Notification experiment + */ +object MRBoldTitleFavoriteAndRetweetExperimentEnum extends Enumeration { + val ShortTitle = Value +} + +/** + * Enum for ML filtering predicates + */ +object QualityPredicateEnum extends Enumeration { + val WeightedOpenOrNtabClick = Value + val ExplicitOpenOrNtabClickFilter = Value + val AlwaysTrue = Value // Disable ML filtering +} + +/** + * Enum to specify normalization used in BigFiltering experiments + */ +object BigFilteringNormalizationEnum extends Enumeration { + val NormalizationDisabled = Value + val NormalizeByNotSendingScore = Value +} + +/** + * Enum for inline actions + */ +object InlineActionsEnum extends Enumeration { + val Favorite = Value + val Follow = Value + val Reply = Value + val Retweet = Value +} + +/** + * Enum for template format + */ +object IbisTemplateFormatEnum extends Enumeration { + val template1 = Value +} + +/** + * Enum for Store name for Top Tweets By Geo + */ +object TopTweetsForGeoCombination extends Enumeration { + val Default = Value + val AccountsTweetFavAsBackfill = Value + val AccountsTweetFavIntermixed = Value +} + +/** + * Enum for scoring function for Top Tweets By Geo + */ +object TopTweetsForGeoRankingFunction extends Enumeration { + val Score = Value + val GeohashLengthAndThenScore = Value +} + +/** + * Enum for which version of popgeo tweets to be using + */ +object PopGeoTweetVersion extends Enumeration { + val Prod = Value +} + +/** + * Enum for Subtext in Android header + */ +object SubtextForAndroidPushHeader extends Enumeration { + val None = Value + val TargetHandler = Value + val TargetTagHandler = Value + val TargetName = Value + val AuthorTagHandler = Value + val AuthorName = Value +} + +object NsfwTextDetectionModel extends Enumeration { + val ProdModel = Value + val RetrainedModel = Value +} + +object HighQualityCandidateGroupEnum extends Enumeration { + val AgeBucket = Value + val Language = Value + val Topic = Value + val Country = Value + val Admin0 = Value + val Admin1 = Value +} + +object CrtGroupEnum extends Enumeration { + val Twistly = Value + val Frs = Value + val F1 = Value + val Topic = Value + val Trip = Value + val GeoPop = Value + val Other = Value + val None = Value +} + +object SportGameEnum extends Enumeration { + val Soccer = Value + val Nfl = Value +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.scala new file mode 100644 index 0000000000..262b3a8b7c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitchParams.scala @@ -0,0 +1,5043 @@ +package com.twitter.frigate.pushservice.params + +import com.twitter.conversions.DurationOps._ +import com.twitter.frigate.pushservice.params.InlineActionsEnum._ +import com.twitter.frigate.pushservice.params.HighQualityCandidateGroupEnum._ +import com.twitter.timelines.configapi.DurationConversion +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.FSEnumParam +import com.twitter.timelines.configapi.FSEnumSeqParam +import com.twitter.timelines.configapi.FSParam +import com.twitter.timelines.configapi.HasDurationConversion +import com.twitter.util.Duration + +object PushFeatureSwitchParams { + + /** + * List of CRTs to uprank. Last CRT in sequence ends up on top of list + */ + object ListOfCrtsToUpRank + extends FSParam[Seq[String]]("rerank_candidates_crt_to_top", default = Seq.empty[String]) + + object ListOfCrtsForOpenApp + extends FSParam[Seq[String]]( + "open_app_allowed_crts", + default = Seq( + "f1firstdegreetweet", + "f1firstdegreephoto", + "f1firstdegreevideo", + "geopoptweet", + "frstweet", + "trendtweet", + "hermituser", + "triangularloopuser" + )) + + /** + * List of CRTs to downrank. Last CRT in sequence ends up on bottom of list + */ + object ListOfCrtsToDownRank + extends FSParam[Seq[String]]( + name = "rerank_candidates_crt_to_downrank", + default = Seq.empty[String]) + + /** + * Param to enable VF filtering in Tweetypie (vs using VisibilityLibrary) + */ + object EnableVFInTweetypie + extends FSParam[Boolean]( + name = "visibility_filtering_enable_vf_in_tweetypie", + default = true + ) + + /** + * Number of max earlybird candidates + */ + object NumberOfMaxEarlybirdInNetworkCandidatesParam + extends FSBoundedParam( + name = "frigate_push_max_earlybird_in_network_candidates", + default = 100, + min = 0, + max = 800 + ) + + /** + * Number of max UserTweetEntityGraph candidates to query + */ + object NumberOfMaxUTEGCandidatesQueriedParam + extends FSBoundedParam( + name = "frigate_push_max_uteg_candidates_queried", + default = 30, + min = 0, + max = 300 + ) + + /** + * Param to control the max tweet age for users + */ + object MaxTweetAgeParam + extends FSBoundedParam[Duration]( + name = "tweet_age_max_hours", + default = 24.hours, + min = 1.hours, + max = 72.hours + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the max tweet age for modeling-based candidates + */ + object ModelingBasedCandidateMaxTweetAgeParam + extends FSBoundedParam[Duration]( + name = "tweet_age_candidate_generation_model_max_hours", + default = 24.hours, + min = 1.hours, + max = 72.hours + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the max tweet age for simcluster-based candidates + */ + object GeoPopTweetMaxAgeInHours + extends FSBoundedParam[Duration]( + name = "tweet_age_geo_pop_max_hours", + default = 24.hours, + min = 1.hours, + max = 120.hours + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the max tweet age for simcluster-based candidates + */ + object SimclusterBasedCandidateMaxTweetAgeParam + extends FSBoundedParam[Duration]( + name = "tweet_age_simcluster_max_hours", + default = 24.hours, + min = 24.hours, + max = 48.hours + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the max tweet age for Detopic-based candidates + */ + object DetopicBasedCandidateMaxTweetAgeParam + extends FSBoundedParam[Duration]( + name = "tweet_age_detopic_max_hours", + default = 24.hours, + min = 24.hours, + max = 48.hours + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the max tweet age for F1 candidates + */ + object F1CandidateMaxTweetAgeParam + extends FSBoundedParam[Duration]( + name = "tweet_age_f1_max_hours", + default = 24.hours, + min = 1.hours, + max = 96.hours + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the max tweet age for Explore Video Tweet + */ + object ExploreVideoTweetAgeParam + extends FSBoundedParam[Duration]( + name = "explore_video_tweets_age_max_hours", + default = 48.hours, + min = 1.hours, + max = 336.hours // Two weeks + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to no send for new user playbook push if user login for past hours + */ + object NewUserPlaybookAllowedLastLoginHours + extends FSBoundedParam[Duration]( + name = "new_user_playbook_allowed_last_login_hours", + default = 0.hours, + min = 0.hours, + max = 72.hours + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * The batch size of RefreshForPushHandler's Take step + */ + object NumberOfMaxCandidatesToBatchInRFPHTakeStep + extends FSBoundedParam( + name = "frigate_push_rfph_batch_take_max_size", + default = 1, + min = 1, + max = 10 + ) + + /** + * The maximum number of candidates to batch for Importance Sampling + */ + object NumberOfMaxCandidatesToBatchForImportanceSampling + extends FSBoundedParam( + name = "frigate_push_rfph_max_candidates_to_batch_for_importance_sampling", + default = 65, + min = 1, + max = 500 + ) + + /** + * Maximum number of regular MR push in 24.hours/daytime/nighttime + */ + object MaxMrPushSends24HoursParam + extends FSBoundedParam( + name = "pushcap_max_sends_24hours", + default = 5, + min = 0, + max = 12 + ) + + /** + * Maximum number of regular MR ntab only channel in 24.hours/daytime/nighttime + */ + object MaxMrNtabOnlySends24HoursParamV3 + extends FSBoundedParam( + name = "pushcap_max_sends_24hours_ntabonly_v3", + default = 5, + min = 0, + max = 12 + ) + + /** + * Maximum number of regular MR ntab only in 24.hours/daytime/nighttime + */ + object MaxMrPushSends24HoursNtabOnlyUsersParam + extends FSBoundedParam( + name = "pushcap_max_sends_24hours_ntab_only", + default = 5, + min = 0, + max = 10 + ) + + /** + * Customized PushCap offset (e.g., to the predicted value) + */ + object CustomizedPushCapOffset + extends FSBoundedParam[Int]( + name = "pushcap_customized_offset", + default = 0, + min = -2, + max = 4 + ) + + /** + * Param to enable restricting minimum pushcap assigned with ML models + * */ + object EnableRestrictedMinModelPushcap + extends FSParam[Boolean]( + name = "pushcap_restricted_model_min_enable", + default = false + ) + + /** + * Param to specify the minimum pushcap allowed to be assigned with ML models + * */ + object RestrictedMinModelPushcap + extends FSBoundedParam[Int]( + name = "pushcap_restricted_model_min_value", + default = 1, + min = 0, + max = 9 + ) + + object EnablePushcapRefactor + extends FSParam[Boolean]( + name = "pushcap_enable_refactor", + default = false + ) + + /** + * Enables the restrict step in pushservice for a given user + * + * Setting this to false may cause a large number of candidates to be passed on to filtering/take + * step in RefreshForPushHandler, increasing the service latency significantly + */ + object EnableRestrictStep extends FSParam[Boolean]("frigate_push_rfph_restrict_step_enable", true) + + /** + * The number of candidates that are able to pass through the restrict step. + */ + object RestrictStepSize + extends FSBoundedParam( + name = "frigate_push_rfph_restrict_step_size", + default = 65, + min = 65, + max = 200 + ) + + /** + * Number of max crMixer candidates to send. + */ + object NumberOfMaxCrMixerCandidatesParam + extends FSBoundedParam( + name = "cr_mixer_migration_max_num_of_candidates_to_return", + default = 400, + min = 0, + max = 2000 + ) + + /** + * Duration between two MR pushes + */ + object MinDurationSincePushParam + extends FSBoundedParam[Duration]( + name = "pushcap_min_duration_since_push_hours", + default = 4.hours, + min = 0.hours, + max = 72.hours + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Each Phase duration to gradually ramp up MagicRecs for new users + */ + object GraduallyRampUpPhaseDurationDays + extends FSBoundedParam[Duration]( + name = "pushcap_gradually_ramp_up_phase_duration_days", + default = 3.days, + min = 2.days, + max = 7.days + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to specify interval for target pushcap fatigue + */ + object TargetPushCapFatigueIntervalHours + extends FSBoundedParam[Duration]( + name = "pushcap_fatigue_interval_hours", + default = 24.hours, + min = 1.hour, + max = 240.hours + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to specify interval for target ntabOnly fatigue + */ + object TargetNtabOnlyCapFatigueIntervalHours + extends FSBoundedParam[Duration]( + name = "pushcap_ntabonly_fatigue_interval_hours", + default = 24.hours, + min = 1.hour, + max = 240.hours + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to use completely explicit push cap instead of LTV/modeling-based + */ + object EnableExplicitPushCap + extends FSParam[Boolean]( + name = "pushcap_explicit_enable", + default = false + ) + + /** + * Param to control explicit push cap (non-LTV) + */ + object ExplicitPushCap + extends FSBoundedParam[Int]( + name = "pushcap_explicit_value", + default = 1, + min = 0, + max = 20 + ) + + /** + * Parameters for percentile thresholds of OpenOrNtabClick model in MR filtering model refreshing DDG + */ + object PercentileThresholdCohort1 + extends FSBoundedParam[Double]( + name = "frigate_push_modeling_percentile_threshold_cohort1", + default = 0.65, + min = 0.0, + max = 1.0 + ) + + object PercentileThresholdCohort2 + extends FSBoundedParam[Double]( + name = "frigate_push_modeling_percentile_threshold_cohort2", + default = 0.03, + min = 0.0, + max = 1.0 + ) + object PercentileThresholdCohort3 + extends FSBoundedParam[Double]( + name = "frigate_push_modeling_percentile_threshold_cohort3", + default = 0.03, + min = 0.0, + max = 1.0 + ) + object PercentileThresholdCohort4 + extends FSBoundedParam[Double]( + name = "frigate_push_modeling_percentile_threshold_cohort4", + default = 0.06, + min = 0.0, + max = 1.0 + ) + object PercentileThresholdCohort5 + extends FSBoundedParam[Double]( + name = "frigate_push_modeling_percentile_threshold_cohort5", + default = 0.06, + min = 0.0, + max = 1.0 + ) + object PercentileThresholdCohort6 + extends FSBoundedParam[Double]( + name = "frigate_push_modeling_percentile_threshold_cohort6", + default = 0.8, + min = 0.0, + max = 1.0 + ) + + /** + * Parameters for percentile threshold list of OpenOrNtabCLick model in MR percentile grid search experiments + */ + object MrPercentileGridSearchThresholdsCohort1 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort1", + default = Seq(0.8, 0.75, 0.65, 0.55, 0.45, 0.35, 0.25) + ) + object MrPercentileGridSearchThresholdsCohort2 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort2", + default = Seq(0.15, 0.12, 0.1, 0.08, 0.06, 0.045, 0.03) + ) + object MrPercentileGridSearchThresholdsCohort3 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort3", + default = Seq(0.15, 0.12, 0.1, 0.08, 0.06, 0.045, 0.03) + ) + object MrPercentileGridSearchThresholdsCohort4 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort4", + default = Seq(0.15, 0.12, 0.1, 0.08, 0.06, 0.045, 0.03) + ) + object MrPercentileGridSearchThresholdsCohort5 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort5", + default = Seq(0.3, 0.2, 0.15, 0.1, 0.08, 0.06, 0.05) + ) + object MrPercentileGridSearchThresholdsCohort6 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_percentile_grid_search_thresholds_cohort6", + default = Seq(0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2) + ) + + /** + * Parameters for threshold list of OpenOrNtabClick model in MF grid search experiments + */ + object MfGridSearchThresholdsCohort1 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_mf_grid_search_thresholds_cohort1", + default = Seq(0.030, 0.040, 0.050, 0.062, 0.070, 0.080, 0.090) // default: 0.062 + ) + object MfGridSearchThresholdsCohort2 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_mf_grid_search_thresholds_cohort2", + default = Seq(0.005, 0.010, 0.015, 0.020, 0.030, 0.040, 0.050) // default: 0.020 + ) + object MfGridSearchThresholdsCohort3 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_mf_grid_search_thresholds_cohort3", + default = Seq(0.010, 0.015, 0.020, 0.025, 0.035, 0.045, 0.055) // default: 0.025 + ) + object MfGridSearchThresholdsCohort4 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_mf_grid_search_thresholds_cohort4", + default = Seq(0.015, 0.020, 0.025, 0.030, 0.040, 0.050, 0.060) // default: 0.030 + ) + object MfGridSearchThresholdsCohort5 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_mf_grid_search_thresholds_cohort5", + default = Seq(0.035, 0.040, 0.045, 0.050, 0.060, 0.070, 0.080) // default: 0.050 + ) + object MfGridSearchThresholdsCohort6 + extends FSParam[Seq[Double]]( + name = "frigate_push_modeling_mf_grid_search_thresholds_cohort6", + default = Seq(0.040, 0.045, 0.050, 0.055, 0.065, 0.075, 0.085) // default: 0.055 + ) + + /** + * Param to specify which global optout models to use to first predict the global scores for users + */ + object GlobalOptoutModelParam + extends FSParam[Seq[OptoutModel.ModelNameType]]( + name = "optout_model_global_model_ids", + default = Seq.empty[OptoutModel.ModelNameType] + ) + + /** + * Param to specify which optout model to use according to the experiment bucket + */ + object BucketOptoutModelParam + extends FSParam[OptoutModel.ModelNameType]( + name = "optout_model_bucket_model_id", + default = OptoutModel.D0_has_realtime_features + ) + + /* + * Param to enable candidate generation model + * */ + object EnableCandidateGenerationModelParam + extends FSParam[Boolean]( + name = "candidate_generation_model_enable", + default = false + ) + + object EnableOverrideForSportsCandidates + extends FSParam[Boolean](name = "magicfanout_sports_event_enable_override", default = true) + + object EnableEventIdBasedOverrideForSportsCandidates + extends FSParam[Boolean]( + name = "magicfanout_sports_event_enable_event_id_based_override", + default = true) + + /** + * Param to specify the threshold to determine if a user’s optout score is high enough to enter the experiment. + */ + object GlobalOptoutThresholdParam + extends FSParam[Seq[Double]]( + name = "optout_model_global_thresholds", + default = Seq(1.0, 1.0) + ) + + /** + * Param to specify the threshold to determine if a user’s optout score is high enough to be assigned + * with a reduced pushcap based on the bucket membership. + */ + object BucketOptoutThresholdParam + extends FSBoundedParam[Double]( + name = "optout_model_bucket_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Param to specify the reduced pushcap value if the optout probability predicted by the bucket + * optout model is higher than the specified bucket optout threshold. + */ + object OptoutExptPushCapParam + extends FSBoundedParam[Int]( + name = "optout_model_expt_push_cap", + default = 10, + min = 0, + max = 10 + ) + + /** + * Param to specify the thresholds to determine which push cap slot the user should be assigned to + * according to the optout score. For example,the slot thresholds are [0.1, 0.2, ..., 1.0], the user + * is assigned to the second slot if the optout score is in (0.1, 0.2]. + */ + object BucketOptoutSlotThresholdParam + extends FSParam[Seq[Double]]( + name = "optout_model_bucket_slot_thresholds", + default = Seq.empty[Double] + ) + + /** + * Param to specify the adjusted push cap of each slot. For example, if the slot push caps are [1, 2, ..., 10] + * and the user is assigned to the 2nd slot according to the optout score, the push cap of the user + * will be adjusted to 2. + */ + object BucketOptoutSlotPushcapParam + extends FSParam[Seq[Int]]( + name = "optout_model_bucket_slot_pushcaps", + default = Seq.empty[Int] + ) + + /** + * Param to specify if the optout score based push cap adjustment is enabled + */ + object EnableOptoutAdjustedPushcap + extends FSParam[Boolean]( + "optout_model_enable_optout_adjusted_pushcap", + false + ) + + /** + * Param to specify which weighted open or ntab click model to use + */ + object WeightedOpenOrNtabClickRankingModelParam + extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( + name = "frigate_push_modeling_oonc_ranking_model_id", + default = WeightedOpenOrNtabClickModel.Periodically_Refreshed_Prod_Model + ) + + /** + * Param to disable heavy ranker + */ + object DisableHeavyRankingModelFSParam + extends FSParam[Boolean]( + name = "frigate_push_modeling_disable_heavy_ranking", + default = false + ) + + /** + * Param to specify which weighted open or ntab click model to use for Android modelling experiment + */ + object WeightedOpenOrNtabClickRankingModelForAndroidParam + extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( + name = "frigate_push_modeling_oonc_ranking_model_for_android_id", + default = WeightedOpenOrNtabClickModel.Periodically_Refreshed_Prod_Model + ) + + /** + * Param to specify which weighted open or ntab click model to use for FILTERING + */ + object WeightedOpenOrNtabClickFilteringModelParam + extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( + name = "frigate_push_modeling_oonc_filtering_model_id", + default = WeightedOpenOrNtabClickModel.Periodically_Refreshed_Prod_Model + ) + + /** + * Param to specify which quality predicate to use for ML filtering + */ + object QualityPredicateIdParam + extends FSEnumParam[QualityPredicateEnum.type]( + name = "frigate_push_modeling_quality_predicate_id", + default = QualityPredicateEnum.WeightedOpenOrNtabClick, + enum = QualityPredicateEnum + ) + + /** + * Param to control threshold for any quality predicates using explicit thresholds + */ + object QualityPredicateExplicitThresholdParam + extends FSBoundedParam[Double]( + name = "frigate_push_modeling_quality_predicate_explicit_threshold", + default = 0.1, + min = 0, + max = 1) + + /** + * MagicFanout relaxed eventID fatigue interval (when we want to enable multiple updates for the same event) + */ + object MagicFanoutRelaxedEventIdFatigueIntervalInHours + extends FSBoundedParam[Int]( + name = "frigate_push_magicfanout_relaxed_event_id_fatigue_interval_in_hours", + default = 24, + min = 0, + max = 720 + ) + + /** + * MagicFanout DenyListed Countries + */ + object MagicFanoutDenyListedCountries + extends FSParam[Seq[String]]( + "frigate_push_magicfanout_denylisted_countries", + Seq.empty[String]) + + object MagicFanoutSportsEventDenyListedCountries + extends FSParam[Seq[String]]( + "magicfanout_sports_event_denylisted_countries", + Seq.empty[String]) + + /** + * MagicFanout maximum erg rank for a given push event for non heavy users + */ + object MagicFanoutRankErgThresholdNonHeavy + extends FSBoundedParam[Int]( + name = "frigate_push_magicfanout_erg_rank_threshold_non_heavy", + default = 25, + min = 1, + max = 50 + ) + + /** + * MagicFanout maximum erg rank for a given push event for heavy users + */ + object MagicFanoutRankErgThresholdHeavy + extends FSBoundedParam[Int]( + name = "frigate_push_magicfanout_erg_rank_threshold_heavy", + default = 20, + min = 1, + max = 50 + ) + + object EnablePushMixerReplacingAllSources + extends FSParam[Boolean]( + name = "push_mixer_enable_replacing_all_sources", + default = false + ) + + object EnablePushMixerReplacingAllSourcesWithControl + extends FSParam[Boolean]( + name = "push_mixer_enable_replacing_all_sources_with_control", + default = false + ) + + object EnablePushMixerReplacingAllSourcesWithExtra + extends FSParam[Boolean]( + name = "push_mixer_enable_replacing_all_sources_with_extra", + default = false + ) + + object EnablePushMixerSource + extends FSParam[Boolean]( + name = "push_mixer_enable_source", + default = false + ) + + object PushMixerMaxResults + extends FSBoundedParam[Int]( + name = "push_mixer_max_results", + default = 10, + min = 1, + max = 5000 + ) + + /** + * Enable tweets from trends that have been annotated by curators + */ + object EnableCuratedTrendTweets + extends FSParam[Boolean](name = "trend_tweet_curated_trends_enable", default = false) + + /** + * Enable tweets from trends that haven't been annotated by curators + */ + object EnableNonCuratedTrendTweets + extends FSParam[Boolean](name = "trend_tweet_non_curated_trends_enable", default = false) + + /** + * Maximum trend tweet notifications in fixed duration + */ + object MaxTrendTweetNotificationsInDuration + extends FSBoundedParam[Int]( + name = "trend_tweet_max_notifications_in_duration", + min = 0, + default = 0, + max = 20) + + /** + * Duration in days over which trend tweet notifications fatigue is applied + */ + object TrendTweetNotificationsFatigueDuration + extends FSBoundedParam[Duration]( + name = "trend_tweet_notifications_fatigue_in_days", + default = 1.day, + min = Duration.Bottom, + max = Duration.Top + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Maximum number of trends candidates to query from event-recos endpoint + */ + object MaxRecommendedTrendsToQuery + extends FSBoundedParam[Int]( + name = "trend_tweet_max_trends_to_query", + min = 0, + default = 0, + max = 100) + + /** + * Fix missing event-associated interests in MagicFanoutNoOptoutInterestsPredicate + */ + object MagicFanoutFixNoOptoutInterestsBugParam + extends FSParam[Boolean]("frigate_push_magicfanout_fix_no_optout_interests", default = true) + + object EnableSimclusterOfflineAggFeatureForExpt + extends FSParam[Boolean]("frigate_enable_simcluster_offline_agg_feature", false) + + /** + * Param to enable removal of UTT domain for + */ + object ApplyMagicFanoutBroadEntityInterestRankThresholdPredicate + extends FSParam[Boolean]( + "frigate_push_magicfanout_broad_entity_interest_rank_threshold_predicate", + false + ) + + object HydrateEventReasonsFeatures + extends FSParam[Boolean]( + name = "frigate_push_magicfanout_hydrate_event_reasons_features", + false + ) + + /** + * Param to enable online MR history features + */ + object EnableHydratingOnlineMRHistoryFeatures + extends FSParam[Boolean]( + name = "feature_hydration_online_mr_history", + default = false + ) + + /** + * Param to enable bold title on favorite and retweet push copy for Android in DDG 10220 + */ + object MRBoldTitleFavoriteAndRetweetParam + extends FSEnumParam[MRBoldTitleFavoriteAndRetweetExperimentEnum.type]( + name = "frigate_push_bold_title_favorite_and_retweet_id", + default = MRBoldTitleFavoriteAndRetweetExperimentEnum.ShortTitle, + enum = MRBoldTitleFavoriteAndRetweetExperimentEnum + ) + + /** + * Param to enable high priority push + */ + object EnableHighPriorityPush + extends FSParam[Boolean]("frigate_push_magicfanout_enable_high_priority_push", false) + + /** + * Param to redirect sports crt event to a custom url + */ + object EnableSearchURLRedirectForSportsFanout + extends FSParam[Boolean]("magicfanout_sports_event_enable_search_url_redirect", false) + + /** + * Param to enable score fanout notification for sports + */ + object EnableScoreFanoutNotification + extends FSParam[Boolean]("magicfanout_sports_event_enable_score_fanout", false) + + /** + * Param to add custom search url for sports crt event + */ + object SearchURLRedirectForSportsFanout + extends FSParam[String]( + name = "magicfanout_sports_event_search_url_redirect", + default = "https://twitter.com/explore/tabs/ipl", + ) + + /** + * Param to enable high priority sports push + */ + object EnableHighPrioritySportsPush + extends FSParam[Boolean]("magicfanout_sports_event_enable_high_priority_push", false) + + /** + * Param to control rank threshold for magicfanout user follow + */ + object MagicFanoutRealgraphRankThreshold + extends FSBoundedParam[Int]( + name = "magicfanout_realgraph_threshold", + default = 500, + max = 500, + min = 100 + ) + + /** + * Topic score threshold for topic proof tweet candidates topic annotations + * */ + object TopicProofTweetCandidatesTopicScoreThreshold + extends FSBoundedParam[Double]( + name = "topics_as_social_proof_topic_score_threshold", + default = 0.0, + min = 0.0, + max = 100.0 + ) + + /** + * Enable Topic Proof Tweet Recs + */ + object EnableTopicProofTweetRecs + extends FSParam[Boolean](name = "topics_as_social_proof_enable", default = true) + + /** + * Enable health filters for topic tweet notifications + */ + object EnableHealthFiltersForTopicProofTweet + extends FSParam[Boolean]( + name = "topics_as_social_proof_enable_health_filters", + default = false) + + /** + * Disable health filters for CrMixer candidates + */ + object DisableHealthFiltersForCrMixerCandidates + extends FSParam[Boolean]( + name = "health_and_quality_filter_disable_for_crmixer_candidates", + default = false) + + object EnableMagicFanoutNewsForYouNtabCopy + extends FSParam[Boolean](name = "send_handler_enable_nfy_ntab_copy", default = false) + + /** + * Param to enable semi-personalized high quality candidates in pushservice + * */ + object HighQualityCandidatesEnableCandidateSource + extends FSParam[Boolean]( + name = "high_quality_candidates_enable_candidate_source", + default = false + ) + + /** + * Param to decide semi-personalized high quality candidates + * */ + object HighQualityCandidatesEnableGroups + extends FSEnumSeqParam[HighQualityCandidateGroupEnum.type]( + name = "high_quality_candidates_enable_groups_ids", + default = Seq(AgeBucket, Language), + enum = HighQualityCandidateGroupEnum + ) + + /** + * Param to decide semi-personalized high quality candidates + * */ + object HighQualityCandidatesNumberOfCandidates + extends FSBoundedParam[Int]( + name = "high_quality_candidates_number_of_candidates", + default = 0, + min = 0, + max = Int.MaxValue + ) + + /** + * Param to enable small domain falling back to bigger domains for high quality candidates in pushservice + * */ + object HighQualityCandidatesEnableFallback + extends FSParam[Boolean]( + name = "high_quality_candidates_enable_fallback", + default = false + ) + + /** + * Param to decide whether to fallback to bigger domain for high quality candidates + * */ + object HighQualityCandidatesMinNumOfCandidatesToFallback + extends FSBoundedParam[Int]( + name = "high_quality_candidates_min_num_of_candidates_to_fallback", + default = 50, + min = 0, + max = Int.MaxValue + ) + + /** + * Param to specific source ids for high quality candidates + * */ + object HighQualityCandidatesFallbackSourceIds + extends FSParam[Seq[String]]( + name = "high_quality_candidates_fallback_source_ids", + default = Seq("HQ_C_COUNT_PASS_QUALITY_SCORES")) + + /** + * Param to decide groups for semi-personalized high quality candidates + * */ + object HighQualityCandidatesFallbackEnabledGroups + extends FSEnumSeqParam[HighQualityCandidateGroupEnum.type]( + name = "high_quality_candidates_fallback_enabled_groups_ids", + default = Seq(Country), + enum = HighQualityCandidateGroupEnum + ) + + /** + * Param to control what heavy ranker model to use for scribing scores + */ + object HighQualityCandidatesHeavyRankingModel + extends FSParam[String]( + name = "high_quality_candidates_heavy_ranking_model", + default = "Periodically_Refreshed_Prod_Model_V11" + ) + + /** + * Param to control what non personalized quality "Cnn" model to use for scribing scores + */ + object HighQualityCandidatesNonPersonalizedQualityCnnModel + extends FSParam[String]( + name = "high_quality_candidates_non_personalized_quality_cnn_model", + default = "Q1_2023_Mr_Tf_Quality_Model_cnn" + ) + + /** + * Param to control what nsfw health model to use for scribing scores + */ + object HighQualityCandidatesBqmlNsfwModel + extends FSParam[String]( + name = "high_quality_candidates_bqml_nsfw_model", + default = "Q2_2022_Mr_Bqml_Health_Model_NsfwV0" + ) + + /** + * Param to control what reportodel to use for scribing scores + */ + object HighQualityCandidatesBqmlReportModel + extends FSParam[String]( + name = "high_quality_candidates_bqml_report_model", + default = "Q3_2022_15266_Mr_Bqml_Non_Personalized_Report_Model_with_Media_Embeddings" + ) + + /** + * Param to specify the threshold to determine if a tweet contains nudity media + */ + object TweetMediaSensitiveCategoryThresholdParam + extends FSBoundedParam[Double]( + name = "tweet_media_sensitive_category_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Param to boost candidates from subscription creators + */ + object BoostCandidatesFromSubscriptionCreators + extends FSParam[Boolean]( + name = "subscription_enable_boost_candidates_from_active_creators", + default = false + ) + + /** + * Param to soft rank candidates from subscription creators + */ + object SoftRankCandidatesFromSubscriptionCreators + extends FSParam[Boolean]( + name = "subscription_enable_soft_rank_candidates_from_active_creators", + default = false + ) + + /** + * Param as factor to control how much we want to boost creator tweets + */ + object SoftRankFactorForSubscriptionCreators + extends FSBoundedParam[Double]( + name = "subscription_soft_rank_factor_for_boost", + default = 1.0, + min = 0.0, + max = Double.MaxValue + ) + + /** + * Param to enable new OON copy for Push Notifications + */ + object EnableNewMROONCopyForPush + extends FSParam[Boolean]( + name = "mr_copy_enable_new_mr_oon_copy_push", + default = true + ) + + /** + * Param to enable generated inline actions on OON Notifications + */ + object EnableOONGeneratedInlineActions + extends FSParam[Boolean]( + name = "mr_inline_enable_oon_generated_actions", + default = false + ) + + /** + * Param to control dynamic inline actions for Out-of-Network copies + */ + object OONTweetDynamicInlineActionsList + extends FSEnumSeqParam[InlineActionsEnum.type]( + name = "mr_inline_oon_tweet_dynamic_action_ids", + default = Seq(Follow, Retweet, Favorite), + enum = InlineActionsEnum + ) + + object HighOONCTweetFormat + extends FSEnumParam[IbisTemplateFormatEnum.type]( + name = "mr_copy_high_oonc_format_id", + default = IbisTemplateFormatEnum.template1, + enum = IbisTemplateFormatEnum + ) + + object LowOONCTweetFormat + extends FSEnumParam[IbisTemplateFormatEnum.type]( + name = "mr_copy_low_oonc_format_id", + default = IbisTemplateFormatEnum.template1, + enum = IbisTemplateFormatEnum + ) + + /** + * Param to enable dynamic inline actions based on FSParams for Tweet copies (not OON) + */ + object EnableTweetDynamicInlineActions + extends FSParam[Boolean]( + name = "mr_inline_enable_tweet_dynamic_actions", + default = false + ) + + /** + * Param to control dynamic inline actions for Tweet copies (not OON) + */ + object TweetDynamicInlineActionsList + extends FSEnumSeqParam[InlineActionsEnum.type]( + name = "mr_inline_tweet_dynamic_action_ids", + default = Seq(Reply, Retweet, Favorite), + enum = InlineActionsEnum + ) + + object UseInlineActionsV1 + extends FSParam[Boolean]( + name = "mr_inline_use_inline_action_v1", + default = true + ) + + object UseInlineActionsV2 + extends FSParam[Boolean]( + name = "mr_inline_use_inline_action_v2", + default = false + ) + + object EnableInlineFeedbackOnPush + extends FSParam[Boolean]( + name = "mr_inline_enable_inline_feedback_on_push", + default = false + ) + + object InlineFeedbackSubstitutePosition + extends FSBoundedParam[Int]( + name = "mr_inline_feedback_substitute_position", + min = 0, + max = 2, + default = 2, // default to substitute or append last inline action + ) + + /** + * Param to control dynamic inline actions for web notifications + */ + object EnableDynamicInlineActionsForDesktopWeb + extends FSParam[Boolean]( + name = "mr_inline_enable_dynamic_actions_for_desktop_web", + default = false + ) + + object EnableDynamicInlineActionsForMobileWeb + extends FSParam[Boolean]( + name = "mr_inline_enable_dynamic_actions_for_mobile_web", + default = false + ) + + /** + * Param to define dynamic inline action types for web notifications (both desktop web + mobile web) + */ + object TweetDynamicInlineActionsListForWeb + extends FSEnumSeqParam[InlineActionsEnum.type]( + name = "mr_inline_tweet_dynamic_action_for_web_ids", + default = Seq(Retweet, Favorite), + enum = InlineActionsEnum + ) + + /** + * Param to enable MR Override Notifications for Android + */ + object EnableOverrideNotificationsForAndroid + extends FSParam[Boolean]( + name = "mr_override_enable_override_notifications_for_android", + default = false + ) + + /** + * Param to enable MR Override Notifications for iOS + */ + object EnableOverrideNotificationsForIos + extends FSParam[Boolean]( + name = "mr_override_enable_override_notifications_for_ios", + default = false + ) + + /** + * Param to enable gradually ramp up notification + */ + object EnableGraduallyRampUpNotification + extends FSParam[Boolean]( + name = "pushcap_gradually_ramp_up_enable", + default = false + ) + + /** + * Param to control the minInrerval for fatigue between consecutive MFNFY pushes + */ + object MFMinIntervalFatigue + extends FSBoundedParam[Duration]( + name = "frigate_push_magicfanout_fatigue_min_interval_consecutive_pushes_minutes", + default = 240.minutes, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromMinutes + } + + /** + * Param to control the interval for MFNFY pushes + */ + object MFPushIntervalInHours + extends FSBoundedParam[Duration]( + name = "frigate_push_magicfanout_fatigue_push_interval_in_hours", + default = 24.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the maximum number of Sports MF pushes in a period of time + */ + object SportsMaxNumberOfPushesInInterval + extends FSBoundedParam[Int]( + name = "magicfanout_sports_event_fatigue_max_pushes_in_interval", + default = 2, + min = 0, + max = 6) + + /** + * Param to control the minInterval for fatigue between consecutive sports pushes + */ + object SportsMinIntervalFatigue + extends FSBoundedParam[Duration]( + name = "magicfanout_sports_event_fatigue_min_interval_consecutive_pushes_minutes", + default = 240.minutes, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromMinutes + } + + /** + * Param to control the interval for sports pushes + */ + object SportsPushIntervalInHours + extends FSBoundedParam[Duration]( + name = "magicfanout_sports_event_fatigue_push_interval_in_hours", + default = 24.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the maximum number of same event sports MF pushes in a period of time + */ + object SportsMaxNumberOfPushesInIntervalPerEvent + extends FSBoundedParam[Int]( + name = "magicfanout_sports_event_fatigue_max_pushes_in_per_event_interval", + default = 2, + min = 0, + max = 6) + + /** + * Param to control the minInterval for fatigue between consecutive same event sports pushes + */ + object SportsMinIntervalFatiguePerEvent + extends FSBoundedParam[Duration]( + name = "magicfanout_sports_event_fatigue_min_interval_consecutive_pushes_per_event_minutes", + default = 240.minutes, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromMinutes + } + + /** + * Param to control the interval for same event sports pushes + */ + object SportsPushIntervalInHoursPerEvent + extends FSBoundedParam[Duration]( + name = "magicfanout_sports_event_fatigue_push_interval_per_event_in_hours", + default = 24.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the maximum number of MF pushes in a period of time + */ + object MFMaxNumberOfPushesInInterval + extends FSBoundedParam[Int]( + name = "frigate_push_magicfanout_fatigue_max_pushes_in_interval", + default = 2, + min = 0, + max = 6) + + /** + * Param to enable custom duration for fatiguing + */ + object GPEnableCustomMagicFanoutCricketFatigue + extends FSParam[Boolean]( + name = "global_participation_cricket_magicfanout_enable_custom_fatigue", + default = false + ) + + /** + * Param to enable e2e scribing for target filtering step + */ + object EnableMrRequestScribingForTargetFiltering + extends FSParam[Boolean]( + name = "mr_request_scribing_enable_for_target_filtering", + default = false + ) + + /** + * Param to enable e2e scribing for candidate filtering step + */ + object EnableMrRequestScribingForCandidateFiltering + extends FSParam[Boolean]( + name = "mr_request_scribing_enable_for_candidate_filtering", + default = false + ) + + /** + * Param to enable e2e scribing with feature hydrating + */ + object EnableMrRequestScribingWithFeatureHydrating + extends FSParam[Boolean]( + name = "mr_request_scribing_enable_with_feature_hydrating", + default = false + ) + + /* + * TargetLevel Feature list for Mr request scribing + */ + object TargetLevelFeatureListForMrRequestScribing + extends FSParam[Seq[String]]( + name = "mr_request_scribing_target_level_feature_list", + default = Seq.empty + ) + + /** + * Param to enable \eps-greedy exploration for BigFiltering/LTV-based filtering + */ + object EnableMrRequestScribingForEpsGreedyExploration + extends FSParam[Boolean]( + name = "mr_request_scribing_eps_greedy_exploration_enable", + default = false + ) + + /** + * Param to control epsilon in \eps-greedy exploration for BigFiltering/LTV-based filtering + */ + object MrRequestScribingEpsGreedyExplorationRatio + extends FSBoundedParam[Double]( + name = "mr_request_scribing_eps_greedy_exploration_ratio", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + /** + * Param to enable scribing dismiss model score + */ + object EnableMrRequestScribingDismissScore + extends FSParam[Boolean]( + name = "mr_request_scribing_dismiss_score_enable", + default = false + ) + + /** + * Param to enable scribing BigFiltering supervised model(s) score(s) + */ + object EnableMrRequestScribingBigFilteringSupervisedScores + extends FSParam[Boolean]( + name = "mr_request_scribing_bigfiltering_supervised_scores_enable", + default = false + ) + + /** + * Param to enable scribing BigFiltering RL model(s) score(s) + */ + object EnableMrRequestScribingBigFilteringRLScores + extends FSParam[Boolean]( + name = "mr_request_scribing_bigfiltering_rl_scores_enable", + default = false + ) + + /** + * Param to flatten mr request scribe + */ + object EnableFlattenMrRequestScribing + extends FSParam[Boolean]( + name = "mr_request_scribing_enable_flatten", + default = false + ) + + /** + * Param to enable NSFW token based filtering + */ + object EnableNsfwTokenBasedFiltering + extends FSParam[Boolean]( + name = "health_and_quality_filter_enable_nsfw_token_based_filtering", + default = false + ) + + object NsfwTokensParam + extends FSParam[Seq[String]]( + name = "health_and_quality_filter_nsfw_tokens", + default = Seq("nsfw", "18+", "\uD83D\uDD1E")) + + object MinimumAllowedAuthorAccountAgeInHours + extends FSBoundedParam[Int]( + name = "health_and_quality_filter_minimum_allowed_author_account_age_in_hours", + default = 0, + min = 0, + max = 168 + ) + + /** + * Param to enable the profanity filter + */ + object EnableProfanityFilterParam + extends FSParam[Boolean]( + name = "health_and_quality_filter_enable_profanity_filter", + default = false + ) + + /** + * Param to enable query the author media representation store + */ + object EnableQueryAuthorMediaRepresentationStore + extends FSParam[Boolean]( + name = "health_and_quality_filter_enable_query_author_media_representation_store", + default = false + ) + + /** + * Threshold to filter a tweet based on the author sensitive media score + */ + object AuthorSensitiveMediaFilteringThreshold + extends FSBoundedParam[Double]( + name = "health_and_quality_filter_author_sensitive_media_filtering_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Threshold to filter a tweet based on the author sensitive media score + */ + object AuthorSensitiveMediaFilteringThresholdForMrTwistly + extends FSBoundedParam[Double]( + name = "health_and_quality_filter_author_sensitive_media_filtering_threshold_for_mrtwistly", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Param to enable filtering the SimCluster tweet if it has AbuseStrike_Top2Percent entitiy + */ + object EnableAbuseStrikeTop2PercentFilterSimCluster + extends FSParam[Boolean]( + name = "health_signal_store_enable_abuse_strike_top_2_percent_filter_sim_cluster", + default = false + ) + + /** + * Param to enable filtering the SimCluster tweet if it has AbuseStrike_Top1Percent entitiy + */ + object EnableAbuseStrikeTop1PercentFilterSimCluster + extends FSParam[Boolean]( + name = "health_signal_store_enable_abuse_strike_top_1_percent_filter_sim_cluster", + default = false + ) + + /** + * Param to enable filtering the SimCluster tweet if it has AbuseStrike_Top0.5Percent entitiy + */ + object EnableAbuseStrikeTop05PercentFilterSimCluster + extends FSParam[Boolean]( + name = "health_signal_store_enable_abuse_strike_top_05_percent_filter_sim_cluster", + default = false + ) + + object EnableAgathaUserHealthModelPredicate + extends FSParam[Boolean]( + name = "health_signal_store_enable_agatha_user_health_model_predicate", + default = false + ) + + /** + * Threshold to filter a tweet based on the agatha_calibrated_nsfw score of its author for MrTwistly + */ + object AgathaCalibratedNSFWThresholdForMrTwistly + extends FSBoundedParam[Double]( + name = "health_signal_store_agatha_calibrated_nsfw_threshold_for_mrtwistly", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Threshold to filter a tweet based on the agatha_calibrated_nsfw score of its author + */ + object AgathaCalibratedNSFWThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_agatha_calibrated_nsfw_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Threshold to filter a tweet based on the agatha_nsfw_text_user score of its author for MrTwistly + */ + object AgathaTextNSFWThresholdForMrTwistly + extends FSBoundedParam[Double]( + name = "health_signal_store_agatha_text_nsfw_threshold_for_mrtwistly", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Threshold to filter a tweet based on the agatha_nsfw_text_user score of its author + */ + object AgathaTextNSFWThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_agatha_text_nsfw_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Threshold to bucket a user based on the agatha_calibrated_nsfw score of the tweet author + */ + object AgathaCalibratedNSFWBucketThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_agatha_calibrated_nsfw_bucket_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Threshold to bucket a user based on the agatha_nsfw_text_user score of the tweet author + */ + object AgathaTextNSFWBucketThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_agatha_text_nsfw_bucket_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Param to enable filtering using pnsfw_text_tweet model. + */ + object EnableHealthSignalStorePnsfwTweetTextPredicate + extends FSParam[Boolean]( + name = "health_signal_store_enable_pnsfw_tweet_text_predicate", + default = false + ) + + /** + * Threshold score for filtering based on pnsfw_text_tweet Model. + */ + object PnsfwTweetTextThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_pnsfw_tweet_text_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Threshold score for bucketing based on pnsfw_text_tweet Model. + */ + object PnsfwTweetTextBucketingThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_pnsfw_tweet_text_bucketing_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Enable filtering tweets with media based on pnsfw_media_tweet Model for OON tweets only. + */ + object PnsfwTweetMediaFilterOonOnly + extends FSParam[Boolean]( + name = "health_signal_store_pnsfw_tweet_media_filter_oon_only", + default = true + ) + + /** + * Threshold score for filtering tweets with media based on pnsfw_media_tweet Model. + */ + object PnsfwTweetMediaThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_pnsfw_tweet_media_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Threshold score for filtering tweets with images based on pnsfw_media_tweet Model. + */ + object PnsfwTweetImageThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_pnsfw_tweet_image_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Threshold score for filtering quote/reply tweets based on source tweet's media + */ + object PnsfwQuoteTweetThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_pnsfw_quote_tweet_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Threshold score for bucketing based on pnsfw_media_tweet Model. + */ + object PnsfwTweetMediaBucketingThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_pnsfw_tweet_media_bucketing_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Param to enable filtering using multilingual psnfw predicate + */ + object EnableHealthSignalStoreMultilingualPnsfwTweetTextPredicate + extends FSParam[Boolean]( + name = "health_signal_store_enable_multilingual_pnsfw_tweet_text_predicate", + default = false + ) + + /** + * Language sequence we will query pnsfw scores for + */ + object MultilingualPnsfwTweetTextSupportedLanguages + extends FSParam[Seq[String]]( + name = "health_signal_store_multilingual_pnsfw_tweet_supported_languages", + default = Seq.empty[String], + ) + + /** + * Threshold score per language for bucketing based on pnsfw scores. + */ + object MultilingualPnsfwTweetTextBucketingThreshold + extends FSParam[Seq[Double]]( + name = "health_signal_store_multilingual_pnsfw_tweet_text_bucketing_thresholds", + default = Seq.empty[Double], + ) + + /** + * Threshold score per language for filtering based on pnsfw scores. + */ + object MultilingualPnsfwTweetTextFilteringThreshold + extends FSParam[Seq[Double]]( + name = "health_signal_store_multilingual_pnsfw_tweet_text_filtering_thresholds", + default = Seq.empty[Double], + ) + + /** + * List of models to threshold scores for bucketing purposes + */ + object MultilingualPnsfwTweetTextBucketingModelList + extends FSEnumSeqParam[NsfwTextDetectionModel.type]( + name = "health_signal_store_multilingual_pnsfw_tweet_text_bucketing_models_ids", + default = Seq(NsfwTextDetectionModel.ProdModel), + enum = NsfwTextDetectionModel + ) + + object MultilingualPnsfwTweetTextModel + extends FSEnumParam[NsfwTextDetectionModel.type]( + name = "health_signal_store_multilingual_pnsfw_tweet_text_model", + default = NsfwTextDetectionModel.ProdModel, + enum = NsfwTextDetectionModel + ) + + /** + * Param to determine media should be enabled for android + */ + object EnableEventSquareMediaAndroid + extends FSParam[Boolean]( + name = "mr_enable_event_media_square_android", + default = false + ) + + /** + * Param to determine expanded media should be enabled for android + */ + object EnableEventPrimaryMediaAndroid + extends FSParam[Boolean]( + name = "mr_enable_event_media_primary_android", + default = false + ) + + /** + * Param to determine media should be enabled for ios for MagicFanout + */ + object EnableEventSquareMediaIosMagicFanoutNewsEvent + extends FSParam[Boolean]( + name = "mr_enable_event_media_square_ios_mf", + default = false + ) + + /** + * Param to configure HTL Visit fatigue + */ + object HTLVisitFatigueTime + extends FSBoundedParam[Int]( + name = "frigate_push_htl_visit_fatigue_time", + default = 20, + min = 0, + max = 72) { + + // Fatigue duration for HTL visit + final val DefaultHoursToFatigueAfterHtlVisit = 20 + final val OldHoursToFatigueAfterHtlVisit = 8 + } + + object MagicFanoutNewsUserGeneratedEventsEnable + extends FSParam[Boolean]( + name = "magicfanout_news_user_generated_events_enable", + default = false) + + object MagicFanoutSkipAccountCountryPredicate + extends FSParam[Boolean]("magicfanout_news_skip_account_country_predicate", false) + + object MagicFanoutNewsEnableDescriptionCopy + extends FSParam[Boolean](name = "magicfanout_news_enable_description_copy", default = false) + + /** + * Enables Custom Targeting for MagicFnaout News events in Pushservice + */ + object MagicFanoutEnableCustomTargetingNewsEvent + extends FSParam[Boolean]("magicfanout_news_event_custom_targeting_enable", false) + + /** + * Enable Topic Copy in MF + */ + object EnableTopicCopyForMF + extends FSParam[Boolean]( + name = "magicfanout_enable_topic_copy", + default = false + ) + + /** + * Enable Topic Copy in MF for implicit topics + */ + object EnableTopicCopyForImplicitTopics + extends FSParam[Boolean]( + name = "magicfanout_enable_topic_copy_erg_interests", + default = false + ) + + /** + * Enable NewCreator push + */ + object EnableNewCreatorPush + extends FSParam[Boolean]( + name = "new_creator_enable_push", + default = false + ) + + /** + * Enable CreatorSubscription push + */ + object EnableCreatorSubscriptionPush + extends FSParam[Boolean]( + name = "creator_subscription_enable_push", + default = false + ) + + /** + * Featureswitch param to enable/disable push recommendations + */ + object EnablePushRecommendationsParam + extends FSParam[Boolean](name = "push_recommendations_enabled", default = false) + + object DisableMlInFilteringFeatureSwitchParam + extends FSParam[Boolean]( + name = "frigate_push_modeling_disable_ml_in_filtering", + default = false + ) + + object EnableMinDurationModifier + extends FSParam[Boolean]( + name = "min_duration_modifier_enable_hour_modifier", + default = false + ) + + object EnableMinDurationModifierV2 + extends FSParam[Boolean]( + name = "min_duration_modifier_enable_hour_modifier_v2", + default = false + ) + + object MinDurationModifierStartHourList + extends FSParam[Seq[Int]]( + name = "min_duration_modifier_start_time_list", + default = Seq(), + ) + + object MinDurationModifierEndHourList + extends FSParam[Seq[Int]]( + name = "min_duration_modifier_start_end_list", + default = Seq(), + ) + + object MinDurationTimeModifierConst + extends FSParam[Seq[Int]]( + name = "min_duration_modifier_const_list", + default = Seq(), + ) + + object EnableQueryUserOpenedHistory + extends FSParam[Boolean]( + name = "min_duration_modifier_enable_query_user_opened_history", + default = false + ) + + object EnableMinDurationModifierByUserHistory + extends FSParam[Boolean]( + name = "min_duration_modifier_enable_hour_modifier_by_user_history", + default = false + ) + + object EnableRandomHourForQuickSend + extends FSParam[Boolean]( + name = "min_duration_modifier_enable_random_hour_for_quick_send", + default = false + ) + + object SendTimeByUserHistoryMaxOpenedThreshold + extends FSBoundedParam[Int]( + name = "min_duration_modifier_max_opened_threshold", + default = 4, + min = 0, + max = 100) + + object SendTimeByUserHistoryNoSendsHours + extends FSBoundedParam[Int]( + name = "min_duration_modifier_no_sends_hours", + default = 1, + min = 0, + max = 24) + + object SendTimeByUserHistoryQuickSendBeforeHours + extends FSBoundedParam[Int]( + name = "min_duration_modifier_quick_send_before_hours", + default = 0, + min = 0, + max = 24) + + object SendTimeByUserHistoryQuickSendAfterHours + extends FSBoundedParam[Int]( + name = "min_duration_modifier_quick_send_after_hours", + default = 0, + min = 0, + max = 24) + + object SendTimeByUserHistoryQuickSendMinDurationInMinute + extends FSBoundedParam[Int]( + name = "min_duration_modifier_quick_send_min_duration", + default = 0, + min = 0, + max = 1440) + + object SendTimeByUserHistoryNoSendMinDuration + extends FSBoundedParam[Int]( + name = "min_duration_modifier_no_send_min_duration", + default = 24, + min = 0, + max = 24) + + object EnableMfGeoTargeting + extends FSParam[Boolean]( + name = "frigate_push_magicfanout_geo_targeting_enable", + default = false) + + /** + * Enable RUX Tweet landing page for push open. When this param is enabled, user will go to RUX + * landing page instead of Tweet details page when opening MagicRecs push. + */ + object EnableRuxLandingPage + extends FSParam[Boolean](name = "frigate_push_enable_rux_landing_page", default = false) + + /** + * Enable RUX Tweet landing page for Ntab Click. When this param is enabled, user will go to RUX + * landing page instead of Tweet details page when click MagicRecs entry on Ntab. + */ + object EnableNTabRuxLandingPage + extends FSParam[Boolean](name = "frigate_push_enable_ntab_rux_landing_page", default = false) + + /** + * Param to enable Onboarding Pushes + */ + object EnableOnboardingPushes + extends FSParam[Boolean]( + name = "onboarding_push_enable", + default = false + ) + + /** + * Param to enable Address Book Pushes + */ + object EnableAddressBookPush + extends FSParam[Boolean]( + name = "onboarding_push_enable_address_book_push", + default = false + ) + + /** + * Param to enable Complete Onboarding Pushes + */ + object EnableCompleteOnboardingPush + extends FSParam[Boolean]( + name = "onboarding_push_enable_complete_onboarding_push", + default = false + ) + + /** + * Param to enable Smart Push Config for MR Override Notifs on Android + */ + object EnableOverrideNotificationsSmartPushConfigForAndroid + extends FSParam[Boolean]( + name = "mr_override_enable_smart_push_config_for_android", + default = false) + + /** + * Param to control the min duration since last MR push for Onboarding Pushes + */ + object MrMinDurationSincePushForOnboardingPushes + extends FSBoundedParam[Duration]( + name = "onboarding_push_min_duration_since_push_days", + default = 4.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to control the push fatigue for Onboarding Pushes + */ + object FatigueForOnboardingPushes + extends FSBoundedParam[Duration]( + name = "onboarding_push_fatigue_days", + default = 30.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to specify the maximum number of Onboarding Push Notifs in a specified period of time + */ + object MaxOnboardingPushInInterval + extends FSBoundedParam[Int]( + name = "onboarding_push_max_in_interval", + default = 1, + min = 0, + max = 10 + ) + + /** + * Param to disable the Onboarding Push Notif Fatigue + */ + object DisableOnboardingPushFatigue + extends FSParam[Boolean]( + name = "onboarding_push_disable_push_fatigue", + default = false + ) + + /** + * Param to control the inverter for fatigue between consecutive TopTweetsByGeoPush + */ + object TopTweetsByGeoPushInterval + extends FSBoundedParam[Duration]( + name = "top_tweets_by_geo_interval_days", + default = 0.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to control the inverter for fatigue between consecutive TripTweets + */ + object HighQualityTweetsPushInterval + extends FSBoundedParam[Duration]( + name = "high_quality_candidates_push_interval_days", + default = 1.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Expiry TTL duration for Tweet Notification types written to history store + */ + object FrigateHistoryTweetNotificationWriteTtl + extends FSBoundedParam[Duration]( + name = "frigate_notification_history_tweet_write_ttl_days", + default = 60.days, + min = Duration.Bottom, + max = Duration.Top + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Expiry TTL duration for Notification written to history store + */ + object FrigateHistoryOtherNotificationWriteTtl + extends FSBoundedParam[Duration]( + name = "frigate_notification_history_other_write_ttl_days", + default = 90.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to control maximum number of TopTweetsByGeoPush pushes to receive in an interval + */ + object MaxTopTweetsByGeoPushGivenInterval + extends FSBoundedParam[Int]( + name = "top_tweets_by_geo_push_given_interval", + default = 1, + min = 0, + max = 10 + ) + + /** + * Param to control maximum number of HighQualityTweet pushes to receive in an interval + */ + object MaxHighQualityTweetsPushGivenInterval + extends FSBoundedParam[Int]( + name = "high_quality_candidates_max_push_given_interval", + default = 3, + min = 0, + max = 10 + ) + + /** + * Param to downrank/backfill top tweets by geo candidates + */ + object BackfillRankTopTweetsByGeoCandidates + extends FSParam[Boolean]( + name = "top_tweets_by_geo_backfill_rank", + default = false + ) + + /** + * Determine whether to use aggressive thresholds for Health filtering on SearchTweet + */ + object PopGeoTweetEnableAggressiveThresholds + extends FSParam[Boolean]( + name = "top_tweets_by_geo_enable_aggressive_health_thresholds", + default = false + ) + + /** + * Param to apply different scoring functions to select top tweets by geo candidates + */ + object ScoringFuncForTopTweetsByGeo + extends FSParam[String]( + name = "top_tweets_by_geo_scoring_function", + default = "Pop8H", + ) + + /** + * Param to query different stores in pop geo service. + */ + object TopTweetsByGeoCombinationParam + extends FSEnumParam[TopTweetsForGeoCombination.type]( + name = "top_tweets_by_geo_combination_id", + default = TopTweetsForGeoCombination.Default, + enum = TopTweetsForGeoCombination + ) + + /** + * Param for popgeo tweet version + */ + object PopGeoTweetVersionParam + extends FSEnumParam[PopGeoTweetVersion.type]( + name = "top_tweets_by_geo_version_id", + default = PopGeoTweetVersion.Prod, + enum = PopGeoTweetVersion + ) + + /** + * Param to query what length of hash for geoh store + */ + object GeoHashLengthList + extends FSParam[Seq[Int]]( + name = "top_tweets_by_geo_hash_length_list", + default = Seq(4), + ) + + /** + * Param to include country code results as back off . + */ + object EnableCountryCodeBackoffTopTweetsByGeo + extends FSParam[Boolean]( + name = "top_tweets_by_geo_enable_country_code_backoff", + default = false, + ) + + /** + * Param to decide ranking function for fetched top tweets by geo + */ + object RankingFunctionForTopTweetsByGeo + extends FSEnumParam[TopTweetsForGeoRankingFunction.type]( + name = "top_tweets_by_geo_ranking_function_id", + default = TopTweetsForGeoRankingFunction.Score, + enum = TopTweetsForGeoRankingFunction + ) + + /** + * Param to enable top tweets by geo candidates + */ + object EnableTopTweetsByGeoCandidates + extends FSParam[Boolean]( + name = "top_tweets_by_geo_enable_candidate_source", + default = false + ) + + /** + * Param to enable top tweets by geo candidates for dormant users + */ + object EnableTopTweetsByGeoCandidatesForDormantUsers + extends FSParam[Boolean]( + name = "top_tweets_by_geo_enable_candidate_source_dormant_users", + default = false + ) + + /** + * Param to specify the maximum number of Top Tweets by Geo candidates to take + */ + object MaxTopTweetsByGeoCandidatesToTake + extends FSBoundedParam[Int]( + name = "top_tweets_by_geo_candidates_to_take", + default = 10, + min = 0, + max = 100 + ) + + /** + * Param to min duration since last MR push for top tweets by geo pushes + */ + object MrMinDurationSincePushForTopTweetsByGeoPushes + extends FSBoundedParam[Duration]( + name = "top_tweets_by_geo_min_duration_since_last_mr_days", + default = 3.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to enable FRS candidate tweets + */ + object EnableFrsCandidates + extends FSParam[Boolean]( + name = "frs_tweet_candidate_enable_adaptor", + default = false + ) + + /** + * Param to enable FRSTweet candidates for topic setting users + * */ + object EnableFrsTweetCandidatesTopicSetting + extends FSParam[Boolean]( + name = "frs_tweet_candidate_enable_adaptor_for_topic_setting", + default = false + ) + + /** + * Param to enable topic annotations for FRSTweet candidates tweets + * */ + object EnableFrsTweetCandidatesTopicAnnotation + extends FSParam[Boolean]( + name = "frs_tweet_candidate_enable_topic_annotation", + default = false + ) + + /** + * Param to enable topic copy for FRSTweet candidates tweets + * */ + object EnableFrsTweetCandidatesTopicCopy + extends FSParam[Boolean]( + name = "frs_tweet_candidate_enable_topic_copy", + default = false + ) + + /** + * Topic score threshold for FRSTweet candidates topic annotations + * */ + object FrsTweetCandidatesTopicScoreThreshold + extends FSBoundedParam[Double]( + name = "frs_tweet_candidate_topic_score_threshold", + default = 0.0, + min = 0.0, + max = 100.0 + ) + + /** + * Param to enable mr modeling-based candidates tweets + * */ + object EnableMrModelingBasedCandidates + extends FSParam[Boolean]( + name = "candidate_generation_model_enable_adaptor", + default = false + ) + + /** + Param to enable mr modeling-based candidates tweets for topic setting users + * */ + object EnableMrModelingBasedCandidatesTopicSetting + extends FSParam[Boolean]( + name = "candidate_generation_model_enable_adaptor_for_topic_setting", + default = false + ) + + /** + * Param to enable topic annotations for mr modeling-based candidates tweets + * */ + object EnableMrModelingBasedCandidatesTopicAnnotation + extends FSParam[Boolean]( + name = "candidate_generation_model_enable_adaptor_topic_annotation", + default = false + ) + + /** + * Topic score threshold for mr modeling based candidates topic annotations + * */ + object MrModelingBasedCandidatesTopicScoreThreshold + extends FSBoundedParam[Double]( + name = "candidate_generation_model_topic_score_threshold", + default = 0.0, + min = 0.0, + max = 100.0 + ) + + /** + * Param to enable topic copy for mr modeling-based candidates tweets + * */ + object EnableMrModelingBasedCandidatesTopicCopy + extends FSParam[Boolean]( + name = "candidate_generation_model_enable_topic_copy", + default = false + ) + + /** + * Number of max mr modeling based candidates + * */ + object NumberOfMaxMrModelingBasedCandidates + extends FSBoundedParam[Int]( + name = "candidate_generation_model_max_mr_modeling_based_candidates", + default = 200, + min = 0, + max = 1000 + ) + + /** + * Enable the traffic to use fav threshold + * */ + object EnableThresholdOfFavMrModelingBasedCandidates + extends FSParam[Boolean]( + name = "candidate_generation_model_enable_fav_threshold", + default = false + ) + + /** + * Threshold of fav for mr modeling based candidates + * */ + object ThresholdOfFavMrModelingBasedCandidates + extends FSBoundedParam[Int]( + name = "candidate_generation_model_fav_threshold", + default = 0, + min = 0, + max = 500 + ) + + /** + * Filtered threshold for mr modeling based candidates + * */ + object CandidateGenerationModelCosineThreshold + extends FSBoundedParam[Double]( + name = "candidate_generation_model_cosine_threshold", + default = 0.9, + min = 0.0, + max = 1.0 + ) + + /* + * ANN hyparameters + * */ + object ANNEfQuery + extends FSBoundedParam[Int]( + name = "candidate_generation_model_ann_ef_query", + default = 300, + min = 50, + max = 1500 + ) + + /** + * Param to do real A/B impression for FRS candidates to avoid dilution + */ + object EnableResultFromFrsCandidates + extends FSParam[Boolean]( + name = "frs_tweet_candidate_enable_returned_result", + default = false + ) + + /** + * Param to enable hashspace candidate tweets + */ + object EnableHashspaceCandidates + extends FSParam[Boolean]( + name = "hashspace_candidate_enable_adaptor", + default = false + ) + + /** + * Param to enable hashspace candidates tweets for topic setting users + * */ + object EnableHashspaceCandidatesTopicSetting + extends FSParam[Boolean]( + name = "hashspace_candidate_enable_adaptor_for_topic_setting", + default = false + ) + + /** + * Param to enable topic annotations for hashspace candidates tweets + * */ + object EnableHashspaceCandidatesTopicAnnotation + extends FSParam[Boolean]( + name = "hashspace_candidate_enable_topic_annotation", + default = false + ) + + /** + * Param to enable topic copy for hashspace candidates tweets + * */ + object EnableHashspaceCandidatesTopicCopy + extends FSParam[Boolean]( + name = "hashspace_candidate_enable_topic_copy", + default = false + ) + + /** + * Topic score threshold for hashspace candidates topic annotations + * */ + object HashspaceCandidatesTopicScoreThreshold + extends FSBoundedParam[Double]( + name = "hashspace_candidate_topic_score_threshold", + default = 0.0, + min = 0.0, + max = 100.0 + ) + + /** + * Param to do real A/B impression for hashspace candidates to avoid dilution + */ + object EnableResultFromHashspaceCandidates + extends FSParam[Boolean]( + name = "hashspace_candidate_enable_returned_result", + default = false + ) + + /** + * Param to enable detopic tweet candidates in adaptor + */ + object EnableDeTopicTweetCandidates + extends FSParam[Boolean]( + name = "detopic_tweet_candidate_enable_adaptor", + default = false + ) + + /** + * Param to enable detopic tweet candidates results (to avoid dilution) + */ + object EnableDeTopicTweetCandidateResults + extends FSParam[Boolean]( + name = "detopic_tweet_candidate_enable_results", + default = false + ) + + /** + * Param to specify whether to provide a custom list of topics in request + */ + object EnableDeTopicTweetCandidatesCustomTopics + extends FSParam[Boolean]( + name = "detopic_tweet_candidate_enable_custom_topics", + default = false + ) + + /** + * Param to specify whether to provide a custom language in request + */ + object EnableDeTopicTweetCandidatesCustomLanguages + extends FSParam[Boolean]( + name = "detopic_tweet_candidate_enable_custom_languages", + default = false + ) + + /** + * Number of detopic tweet candidates in the request + * */ + object NumberOfDeTopicTweetCandidates + extends FSBoundedParam[Int]( + name = "detopic_tweet_candidate_num_candidates_in_request", + default = 600, + min = 0, + max = 3000 + ) + + /** + * Max Number of detopic tweet candidates returned in adaptor + * */ + object NumberOfMaxDeTopicTweetCandidatesReturned + extends FSBoundedParam[Int]( + name = "detopic_tweet_candidate_max_num_candidates_returned", + default = 200, + min = 0, + max = 3000 + ) + + /** + * Param to enable F1 from protected Authors + */ + object EnableF1FromProtectedTweetAuthors + extends FSParam[Boolean]( + "f1_enable_protected_tweets", + false + ) + + /** + * Param to enable safe user tweet tweetypie store + */ + object EnableSafeUserTweetTweetypieStore + extends FSParam[Boolean]( + "mr_infra_enable_use_safe_user_tweet_tweetypie", + false + ) + + /** + * Param to min duration since last MR push for top tweets by geo pushes + */ + object EnableMrMinDurationSinceMrPushFatigue + extends FSParam[Boolean]( + name = "top_tweets_by_geo_enable_min_duration_since_mr_fatigue", + default = false + ) + + /** + * Param to check time since last time user logged in for geo top tweets by geo push + */ + object TimeSinceLastLoginForGeoPopTweetPush + extends FSBoundedParam[Duration]( + name = "top_tweets_by_geo_time_since_last_login_in_days", + default = 14.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to check time since last time user logged in for geo top tweets by geo push + */ + object MinimumTimeSinceLastLoginForGeoPopTweetPush + extends FSBoundedParam[Duration]( + name = "top_tweets_by_geo_minimum_time_since_last_login_in_days", + default = 14.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** How long we wait after a user visited the app before sending them a space fanout rec */ + object SpaceRecsAppFatigueDuration + extends FSBoundedParam[Duration]( + name = "space_recs_app_fatigue_duration_hours", + default = 4.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** The fatigue time-window for OON space fanout recs, e.g. 1 push every 3 days */ + object OONSpaceRecsFatigueDuration + extends FSBoundedParam[Duration]( + name = "space_recs_oon_fatigue_duration_days", + default = 1.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** The global fatigue time-window for space fanout recs, e.g. 1 push every 3 days */ + object SpaceRecsGlobalFatigueDuration + extends FSBoundedParam[Duration]( + name = "space_recs_global_fatigue_duration_days", + default = 1.day, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** The min-interval between space fanout recs. + * After receiving a space fanout rec, they must wait a minimum of this + * interval before eligibile for another */ + object SpaceRecsFatigueMinIntervalDuration + extends FSBoundedParam[Duration]( + name = "space_recs_fatigue_mininterval_duration_minutes", + default = 30.minutes, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromMinutes + } + + /** Space fanout user-follow rank threshold. + * Users targeted by a follow that is above this threshold will be filtered */ + object SpaceRecsRealgraphThreshold + extends FSBoundedParam[Int]( + name = "space_recs_realgraph_threshold", + default = 50, + max = 500, + min = 0 + ) + + object EnableHydratingRealGraphTargetUserFeatures + extends FSParam[Boolean]( + name = "frigate_push_modeling_enable_hydrating_real_graph_target_user_feature", + default = true + ) + + /** Param to reduce dillution when checking if a space is featured or not */ + object CheckFeaturedSpaceOON + extends FSParam[Boolean](name = "space_recs_check_if_its_featured_space", default = false) + + /** Enable Featured Spaces Rules for OON spaces */ + object EnableFeaturedSpacesOON + extends FSParam[Boolean](name = "space_recs_enable_featured_spaces_oon", default = false) + + /** Enable Geo Targeting */ + object EnableGeoTargetingForSpaces + extends FSParam[Boolean](name = "space_recs_enable_geo_targeting", default = false) + + /** Number of max pushes within the fatigue duration for OON Space Recs */ + object OONSpaceRecsPushLimit + extends FSBoundedParam[Int]( + name = "space_recs_oon_push_limit", + default = 1, + max = 3, + min = 0 + ) + + /** Space fanout recs, number of max pushes within the fatigue duration */ + object SpaceRecsGlobalPushLimit + extends FSBoundedParam[Int]( + name = "space_recs_global_push_limit", + default = 3, + max = 50, + min = 0 + ) + + /** + * Param to enable score based override. + */ + object EnableOverrideNotificationsScoreBasedOverride + extends FSParam[Boolean]( + name = "mr_override_enable_score_ranking", + default = false + ) + + /** + * Param to determine the lookback duration when searching for override info. + */ + object OverrideNotificationsLookbackDurationForOverrideInfo + extends FSBoundedParam[Duration]( + name = "mr_override_lookback_duration_override_info_in_days", + default = 30.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to determine the lookback duration when searching for impression ids. + */ + object OverrideNotificationsLookbackDurationForImpressionId + extends FSBoundedParam[Duration]( + name = "mr_override_lookback_duration_impression_id_in_days", + default = 30.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to enable sending multiple target ids in the payload. + */ + object EnableOverrideNotificationsMultipleTargetIds + extends FSParam[Boolean]( + name = "mr_override_enable_multiple_target_ids", + default = false + ) + + /** + * Param for MR Web Notifications holdback + */ + object MRWebHoldbackParam + extends FSParam[Boolean]( + name = "mr_web_notifications_holdback", + default = false + ) + + object CommonRecommendationTypeDenyListPushHoldbacks + extends FSParam[Seq[String]]( + name = "crt_to_exclude_from_holdbacks_push_holdbacks", + default = Seq.empty[String] + ) + + /** + * Param to enable sending number of slots to maintain in the payload. + */ + object EnableOverrideNotificationsNSlots + extends FSParam[Boolean]( + name = "mr_override_enable_n_slots", + default = false + ) + + /** + * Enable down ranking of NUPS and pop geo topic follow candidates for new user playbook. + */ + object EnableDownRankOfNewUserPlaybookTopicFollowPush + extends FSParam[Boolean]( + name = "topic_follow_new_user_playbook_enable_down_rank", + default = false + ) + + /** + * Enable down ranking of NUPS and pop geo topic tweet candidates for new user playbook. + */ + object EnableDownRankOfNewUserPlaybookTopicTweetPush + extends FSParam[Boolean]( + name = "topic_tweet_new_user_playbook_enable_down_rank", + default = false + ) + + /** + * Param to enable/disable employee only spaces for fanout of notifications + */ + object EnableEmployeeOnlySpaceNotifications + extends FSParam[Boolean](name = "space_recs_employee_only_enable", default = false) + + /** + * NTab spaces ttl experiments + */ + object EnableSpacesTtlForNtab + extends FSParam[Boolean]( + name = "ntab_spaces_ttl_enable", + default = false + ) + + /** + * Param to determine the ttl duration for space notifications on NTab. + */ + object SpaceNotificationsTTLDurationForNTab + extends FSBoundedParam[Duration]( + name = "ntab_spaces_ttl_hours", + default = 1.hour, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /* + * NTab override experiments + * see go/ntab-override experiment brief for more details + */ + + /** + * Override notifications for Spaces on lockscreen. + */ + object EnableOverrideForSpaces + extends FSParam[Boolean]( + name = "mr_override_spaces", + default = false + ) + + /** + * Param to enable storing the Generic Notification Key. + */ + object EnableStoringNtabGenericNotifKey + extends FSParam[Boolean]( + name = "ntab_enable_storing_generic_notif_key", + default = false + ) + + /** + * Param to enable deleting the Target's timeline. + */ + object EnableDeletingNtabTimeline + extends FSParam[Boolean]( + name = "ntab_enable_delete_timeline", + default = false + ) + + /** + * Param to enable sending the overrideId + * to NTab which enables override support in NTab-api + */ + object EnableOverrideIdNTabRequest + extends FSParam[Boolean]( + name = "ntab_enable_override_id_in_request", + default = false + ) + + /** + * [Override Workstream] Param to enable NTab override n-slot feature. + */ + object EnableNslotsForOverrideOnNtab + extends FSParam[Boolean]( + name = "ntab_enable_override_max_count", + default = false + ) + + /** + * Param to determine the lookback duration for override candidates on NTab. + */ + object OverrideNotificationsLookbackDurationForNTab + extends FSBoundedParam[Duration]( + name = "ntab_override_lookback_duration_days", + default = 30.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to determine the max count for candidates on NTab. + */ + object OverrideNotificationsMaxCountForNTab + extends FSBoundedParam[Int]( + name = "ntab_override_limit", + min = 0, + max = Int.MaxValue, + default = 4) + + //// end override experiments //// + /** + * Param to enable top tweet impressions notification + */ + object EnableTopTweetImpressionsNotification + extends FSParam[Boolean]( + name = "top_tweet_impressions_notification_enable", + default = false + ) + + /** + * Param to control the inverter for fatigue between consecutive TweetImpressions + */ + object TopTweetImpressionsNotificationInterval + extends FSBoundedParam[Duration]( + name = "top_tweet_impressions_notification_interval_days", + default = 7.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * The min-interval between TweetImpressions notifications. + * After receiving a TweetImpressions notif, they must wait a minimum of this + * interval before being eligible for another + */ + object TopTweetImpressionsFatigueMinIntervalDuration + extends FSBoundedParam[Duration]( + name = "top_tweet_impressions_fatigue_mininterval_duration_days", + default = 1.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Maximum number of top tweet impressions notifications to receive in an interval + */ + object MaxTopTweetImpressionsNotifications + extends FSBoundedParam( + name = "top_tweet_impressions_fatigue_max_in_interval", + default = 0, + min = 0, + max = 10 + ) + + /** + * Param for min number of impressions counts to be eligible for lonely_birds_tweet_impressions model + */ + object TopTweetImpressionsMinRequired + extends FSBoundedParam[Int]( + name = "top_tweet_impressions_min_required", + default = 25, + min = 0, + max = Int.MaxValue + ) + + /** + * Param for threshold of impressions counts to notify for lonely_birds_tweet_impressions model + */ + object TopTweetImpressionsThreshold + extends FSBoundedParam[Int]( + name = "top_tweet_impressions_threshold", + default = 25, + min = 0, + max = Int.MaxValue + ) + + /** + * Param for the number of days to search up to for a user's original tweets + */ + object TopTweetImpressionsOriginalTweetsNumDaysSearch + extends FSBoundedParam[Int]( + name = "top_tweet_impressions_original_tweets_num_days_search", + default = 3, + min = 0, + max = 21 + ) + + /** + * Param for the minimum number of original tweets a user needs to be considered an original author + */ + object TopTweetImpressionsMinNumOriginalTweets + extends FSBoundedParam[Int]( + name = "top_tweet_impressions_num_original_tweets", + default = 3, + min = 0, + max = Int.MaxValue + ) + + /** + * Param for the max number of favorites any original Tweet can have + */ + object TopTweetImpressionsMaxFavoritesPerTweet + extends FSBoundedParam[Int]( + name = "top_tweet_impressions_max_favorites_per_tweet", + default = 3, + min = 0, + max = Int.MaxValue + ) + + /** + * Param for the max number of total inbound favorites for a user's tweets + */ + object TopTweetImpressionsTotalInboundFavoritesLimit + extends FSBoundedParam[Int]( + name = "top_tweet_impressions_total_inbound_favorites_limit", + default = 60, + min = 0, + max = Int.MaxValue + ) + + /** + * Param for the number of days to search for tweets to count the total inbound favorites + */ + object TopTweetImpressionsTotalFavoritesLimitNumDaysSearch + extends FSBoundedParam[Int]( + name = "top_tweet_impressions_total_favorites_limit_num_days_search", + default = 7, + min = 0, + max = 21 + ) + + /** + * Param for the max number of recent tweets Tflock should return + */ + object TopTweetImpressionsRecentTweetsByAuthorStoreMaxResults + extends FSBoundedParam[Int]( + name = "top_tweet_impressions_recent_tweets_by_author_store_max_results", + default = 50, + min = 0, + max = 1000 + ) + + /* + * Param to represent the max number of slots to maintain for Override Notifications + */ + object OverrideNotificationsMaxNumOfSlots + extends FSBoundedParam[Int]( + name = "mr_override_max_num_slots", + default = 1, + max = 10, + min = 1 + ) + + object EnableOverrideMaxSlotFn + extends FSParam[Boolean]( + name = "mr_override_enable_max_num_slots_fn", + default = false + ) + + object OverrideMaxSlotFnPushCapKnobs + extends FSParam[Seq[Double]]("mr_override_fn_pushcap_knobs", default = Seq.empty[Double]) + + object OverrideMaxSlotFnNSlotKnobs + extends FSParam[Seq[Double]]("mr_override_fn_nslot_knobs", default = Seq.empty[Double]) + + object OverrideMaxSlotFnPowerKnobs + extends FSParam[Seq[Double]]("mr_override_fn_power_knobs", default = Seq.empty[Double]) + + object OverrideMaxSlotFnWeight + extends FSBoundedParam[Double]( + "mr_override_fn_weight", + default = 1.0, + min = 0.0, + max = Double.MaxValue) + + /** + * Use to enable sending target ids in the Smart Push Payload + */ + object EnableTargetIdsInSmartPushPayload + extends FSParam[Boolean](name = "mr_override_enable_target_ids", default = true) + + /** + * Param to enable override by target id for MagicFanoutSportsEvent candidates + */ + object EnableTargetIdInSmartPushPayloadForMagicFanoutSportsEvent + extends FSParam[Boolean]( + name = "mr_override_enable_target_id_for_magic_fanout_sports_event", + default = true) + + /** + * Param to enable secondary account predicate on MF NFY + */ + object EnableSecondaryAccountPredicateMF + extends FSParam[Boolean]( + name = "frigate_push_magicfanout_secondary_account_predicate", + default = false + ) + + /** + * Enables showing our customers videos on their notifications + */ + object EnableInlineVideo + extends FSParam[Boolean](name = "mr_inline_enable_inline_video", default = false) + + /** + * Enables autoplay for inline videos + */ + object EnableAutoplayForInlineVideo + extends FSParam[Boolean](name = "mr_inline_enable_autoplay_for_inline_video", default = false) + + /** + * Enable OON filtering based on MentionFilter. + */ + object EnableOONFilteringBasedOnUserSettings + extends FSParam[Boolean](name = "oon_filtering_enable_based_on_user_settings", false) + + /** + * Enables Custom Thread Ids which is used to ungroup notifications for N-slots on iOS + */ + object EnableCustomThreadIdForOverride + extends FSParam[Boolean](name = "mr_override_enable_custom_thread_id", default = false) + + /** + * Enables showing verified symbol in the push presentation + */ + object EnablePushPresentationVerifiedSymbol + extends FSParam[Boolean](name = "push_presentation_enable_verified_symbol", default = false) + + /** + * Decide subtext in Android push header + */ + object SubtextInAndroidPushHeaderParam + extends FSEnumParam[SubtextForAndroidPushHeader.type]( + name = "push_presentation_subtext_in_android_push_header_id", + default = SubtextForAndroidPushHeader.None, + enum = SubtextForAndroidPushHeader) + + /** + * Enable SimClusters Targeting For Spaces. If false we just drop all candidates with such targeting reason + */ + object EnableSimClusterTargetingSpaces + extends FSParam[Boolean](name = "space_recs_send_simcluster_recommendations", default = false) + + /** + * Param to control threshold for dot product of simcluster based targeting on Spaces + */ + object SpacesTargetingSimClusterDotProductThreshold + extends FSBoundedParam[Double]( + "space_recs_simclusters_dot_product_threshold", + default = 0.0, + min = 0.0, + max = 10.0) + + /** + * Param to control top-k clusters simcluster based targeting on Spaces + */ + object SpacesTopKSimClusterCount + extends FSBoundedParam[Int]( + "space_recs_simclusters_top_k_count", + default = 1, + min = 1, + max = 50) + + /** SimCluster users host/speaker must meet this follower count minimum threshold to be considered for sends */ + object SpaceRecsSimClusterUserMinimumFollowerCount + extends FSBoundedParam[Int]( + name = "space_recs_simcluster_user_min_follower_count", + default = 5000, + max = Int.MaxValue, + min = 0 + ) + + /** + * Target has been bucketed into the Inline Action App Visit Fatigue Experiment + */ + object TargetInInlineActionAppVisitFatigue + extends FSParam[Boolean](name = "inline_action_target_in_app_visit_fatigue", default = false) + + /** + * Enables Inline Action App Visit Fatigue + */ + object EnableInlineActionAppVisitFatigue + extends FSParam[Boolean](name = "inline_action_enable_app_visit_fatigue", default = false) + + /** + * Determines the fatigue that we should apply when the target user has performed an inline action + */ + object InlineActionAppVisitFatigue + extends FSBoundedParam[Duration]( + name = "inline_action_app_visit_fatigue_hours", + default = 8.hours, + min = 1.hour, + max = 48.hours) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Weight for reranking(oonc - weight * nudityRate) + */ + object AuthorSensitiveScoreWeightInReranking + extends FSBoundedParam[Double]( + name = "rerank_candidates_author_sensitive_score_weight_in_reranking", + default = 0.0, + min = -100.0, + max = 100.0 + ) + + /** + * Param to control the last active space listener threshold to filter out based on that + */ + object SpaceParticipantHistoryLastActiveThreshold + extends FSBoundedParam[Duration]( + name = "space_recs_last_active_space_listener_threshold_in_hours", + default = 0.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /* + * Param to enable mr user simcluster feature set (v2020) hydration for modeling-based candidate generation + * */ + object HydrateMrUserSimclusterV2020InModelingBasedCG + extends FSParam[Boolean]( + name = "candidate_generation_model_hydrate_mr_user_simcluster_v2020", + default = false) + + /* + * Param to enable mr semantic core feature set hydration for modeling-based candidate generation + * */ + object HydrateMrUserSemanticCoreInModelingBasedCG + extends FSParam[Boolean]( + name = "candidate_generation_model_hydrate_mr_user_semantic_core", + default = false) + + /* + * Param to enable mr semantic core feature set hydration for modeling-based candidate generation + * */ + object HydrateOnboardingInModelingBasedCG + extends FSParam[Boolean]( + name = "candidate_generation_model_hydrate_onboarding", + default = false) + + /* + * Param to enable mr topic follow feature set hydration for modeling-based candidate generation + * */ + object HydrateTopicFollowInModelingBasedCG + extends FSParam[Boolean]( + name = "candidate_generation_model_hydrate_topic_follow", + default = false) + + /* + * Param to enable mr user topic feature set hydration for modeling-based candidate generation + * */ + object HydrateMrUserTopicInModelingBasedCG + extends FSParam[Boolean]( + name = "candidate_generation_model_hydrate_mr_user_topic", + default = false) + + /* + * Param to enable mr user topic feature set hydration for modeling-based candidate generation + * */ + object HydrateMrUserAuthorInModelingBasedCG + extends FSParam[Boolean]( + name = "candidate_generation_model_hydrate_mr_user_author", + default = false) + + /* + * Param to enable user penguin language feature set hydration for modeling-based candidate generation + * */ + object HydrateUserPenguinLanguageInModelingBasedCG + extends FSParam[Boolean]( + name = "candidate_generation_model_hydrate_user_penguin_language", + default = false) + /* + * Param to enable user geo feature set hydration for modeling-based candidate generation + * */ + object HydrateUseGeoInModelingBasedCG + extends FSParam[Boolean]( + name = "candidate_generation_model_hydrate_user_geo", + default = false) + + /* + * Param to enable mr user hashspace embedding feature set hydration for modeling-based candidate generation + * */ + object HydrateMrUserHashspaceEmbeddingInModelingBasedCG + extends FSParam[Boolean]( + name = "candidate_generation_model_hydrate_mr_user_hashspace_embedding", + default = false) + /* + * Param to enable user tweet text feature hydration + * */ + object EnableMrUserEngagedTweetTokensFeature + extends FSParam[Boolean]( + name = "feature_hydration_mr_user_engaged_tweet_tokens", + default = false) + + /** + * Params for CRT based see less often fatigue rules + */ + object EnableF1TriggerSeeLessOftenFatigue + extends FSParam[Boolean]( + name = "seelessoften_enable_f1_trigger_fatigue", + default = false + ) + + object EnableNonF1TriggerSeeLessOftenFatigue + extends FSParam[Boolean]( + name = "seelessoften_enable_nonf1_trigger_fatigue", + default = false + ) + + /** + * Adjust the NtabCaretClickFatigue for candidates if it is triggered by + * TripHqTweet candidates + */ + object AdjustTripHqTweetTriggeredNtabCaretClickFatigue + extends FSParam[Boolean]( + name = "seelessoften_adjust_trip_hq_tweet_triggered_fatigue", + default = false + ) + + object NumberOfDaysToFilterForSeeLessOftenForF1TriggerF1 + extends FSBoundedParam[Duration]( + name = "seelessoften_for_f1_trigger_f1_tofiltermr_days", + default = 7.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + object NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerF1 + extends FSBoundedParam[Duration]( + name = "seelessoften_for_f1_trigger_f1_toreduce_pushcap_days", + default = 30.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + object NumberOfDaysToFilterForSeeLessOftenForF1TriggerNonF1 + extends FSBoundedParam[Duration]( + name = "seelessoften_for_f1_trigger_nonf1_tofiltermr_days", + default = 7.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + object NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerNonF1 + extends FSBoundedParam[Duration]( + name = "seelessoften_for_f1_trigger_non_f1_toreduce_pushcap_days", + default = 30.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + object NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerF1 + extends FSBoundedParam[Duration]( + name = "seelessoften_for_nonf1_trigger_f1_tofiltermr_days", + default = 7.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + object NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerF1 + extends FSBoundedParam[Duration]( + name = "seelessoften_for_nonf1_trigger_f1_toreduce_pushcap_days", + default = 30.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + object NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerNonF1 + extends FSBoundedParam[Duration]( + name = "seelessoften_for_nonf1_trigger_nonf1_tofiltermr_days", + default = 7.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + object NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerNonF1 + extends FSBoundedParam[Duration]( + name = "seelessoften_for_nonf1_trigger_nonf1_toreduce_pushcap_days", + default = 30.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + object EnableContFnF1TriggerSeeLessOftenFatigue + extends FSParam[Boolean]( + name = "seelessoften_fn_enable_f1_trigger_fatigue", + default = false + ) + + object EnableContFnNonF1TriggerSeeLessOftenFatigue + extends FSParam[Boolean]( + name = "seelessoften_fn_enable_nonf1_trigger_fatigue", + default = false + ) + + object SeeLessOftenListOfDayKnobs + extends FSParam[Seq[Double]]("seelessoften_fn_day_knobs", default = Seq.empty[Double]) + + object SeeLessOftenListOfPushCapWeightKnobs + extends FSParam[Seq[Double]]("seelessoften_fn_pushcap_knobs", default = Seq.empty[Double]) + + object SeeLessOftenListOfPowerKnobs + extends FSParam[Seq[Double]]("seelessoften_fn_power_knobs", default = Seq.empty[Double]) + + object SeeLessOftenF1TriggerF1PushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_f1_trigger_f1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object SeeLessOftenF1TriggerNonF1PushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_f1_trigger_nonf1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object SeeLessOftenNonF1TriggerF1PushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_nonf1_trigger_f1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object SeeLessOftenNonF1TriggerNonF1PushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_nonf1_trigger_nonf1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object SeeLessOftenTripHqTweetTriggerF1PushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_trip_hq_tweet_trigger_f1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object SeeLessOftenTripHqTweetTriggerNonF1PushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_trip_hq_tweet_trigger_nonf1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object SeeLessOftenTripHqTweetTriggerTripHqTweetPushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_trip_hq_tweet_trigger_trip_hq_tweet_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object SeeLessOftenTopicTriggerTopicPushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_topic_trigger_topic_weight", + default = 1.0, + min = 0.0, + max = Double.MaxValue) + + object SeeLessOftenTopicTriggerF1PushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_topic_trigger_f1_weight", + default = 100000.0, + min = 0.0, + max = Double.MaxValue) + + object SeeLessOftenTopicTriggerOONPushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_topic_trigger_oon_weight", + default = 100000.0, + min = 0.0, + max = Double.MaxValue) + + object SeeLessOftenF1TriggerTopicPushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_f1_trigger_topic_weight", + default = 100000.0, + min = 0.0, + max = Double.MaxValue) + + object SeeLessOftenOONTriggerTopicPushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_oon_trigger_topic_weight", + default = 1.0, + min = 0.0, + max = Double.MaxValue) + + object SeeLessOftenDefaultPushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_default_weight", + default = 100000.0, + min = 0.0, + max = Double.MaxValue) + + object SeeLessOftenNtabOnlyNotifUserPushCapWeight + extends FSBoundedParam[Double]( + "seelessoften_fn_ntab_only_user_weight", + default = 1.0, + min = 0.0, + max = Double.MaxValue) + + // Params for inline feedback fatigue + object EnableContFnF1TriggerInlineFeedbackFatigue + extends FSParam[Boolean]( + name = "feedback_inline_fn_enable_f1_trigger_fatigue", + default = false + ) + + object EnableContFnNonF1TriggerInlineFeedbackFatigue + extends FSParam[Boolean]( + name = "feedback_inline_fn_enable_nonf1_trigger_fatigue", + default = false + ) + + object UseInlineDislikeForFatigue + extends FSParam[Boolean]( + name = "feedback_inline_fn_use_dislike", + default = true + ) + object UseInlineDismissForFatigue + extends FSParam[Boolean]( + name = "feedback_inline_fn_use_dismiss", + default = false + ) + object UseInlineSeeLessForFatigue + extends FSParam[Boolean]( + name = "feedback_inline_fn_use_see_less", + default = false + ) + object UseInlineNotRelevantForFatigue + extends FSParam[Boolean]( + name = "feedback_inline_fn_use_not_relevant", + default = false + ) + object InlineFeedbackListOfDayKnobs + extends FSParam[Seq[Double]]("feedback_inline_fn_day_knobs", default = Seq.empty[Double]) + + object InlineFeedbackListOfPushCapWeightKnobs + extends FSParam[Seq[Double]]("feedback_inline_fn_pushcap_knobs", default = Seq.empty[Double]) + + object InlineFeedbackListOfPowerKnobs + extends FSParam[Seq[Double]]("feedback_inline_fn_power_knobs", default = Seq.empty[Double]) + + object InlineFeedbackF1TriggerF1PushCapWeight + extends FSBoundedParam[Double]( + "feedback_inline_fn_f1_trigger_f1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object InlineFeedbackF1TriggerNonF1PushCapWeight + extends FSBoundedParam[Double]( + "feedback_inline_fn_f1_trigger_nonf1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object InlineFeedbackNonF1TriggerF1PushCapWeight + extends FSBoundedParam[Double]( + "feedback_inline_fn_nonf1_trigger_f1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object InlineFeedbackNonF1TriggerNonF1PushCapWeight + extends FSBoundedParam[Double]( + "feedback_inline_fn_nonf1_trigger_nonf1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + // Params for prompt feedback + object EnableContFnF1TriggerPromptFeedbackFatigue + extends FSParam[Boolean]( + name = "feedback_prompt_fn_enable_f1_trigger_fatigue", + default = false + ) + + object EnableContFnNonF1TriggerPromptFeedbackFatigue + extends FSParam[Boolean]( + name = "feedback_prompt_fn_enable_nonf1_trigger_fatigue", + default = false + ) + object PromptFeedbackListOfDayKnobs + extends FSParam[Seq[Double]]("feedback_prompt_fn_day_knobs", default = Seq.empty[Double]) + + object PromptFeedbackListOfPushCapWeightKnobs + extends FSParam[Seq[Double]]("feedback_prompt_fn_pushcap_knobs", default = Seq.empty[Double]) + + object PromptFeedbackListOfPowerKnobs + extends FSParam[Seq[Double]]("feedback_prompt_fn_power_knobs", default = Seq.empty[Double]) + + object PromptFeedbackF1TriggerF1PushCapWeight + extends FSBoundedParam[Double]( + "feedback_prompt_fn_f1_trigger_f1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object PromptFeedbackF1TriggerNonF1PushCapWeight + extends FSBoundedParam[Double]( + "feedback_prompt_fn_f1_trigger_nonf1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object PromptFeedbackNonF1TriggerF1PushCapWeight + extends FSBoundedParam[Double]( + "feedback_prompt_fn_nonf1_trigger_f1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + object PromptFeedbackNonF1TriggerNonF1PushCapWeight + extends FSBoundedParam[Double]( + "feedback_prompt_fn_nonf1_trigger_nonf1_weight", + default = 1.0, + min = 0.0, + max = 10000000.0) + + /* + * Param to enable cohost join event notif + */ + object EnableSpaceCohostJoinEvent + extends FSParam[Boolean](name = "space_recs_cohost_join_enable", default = true) + + /* + * Param to bypass global push cap when target is device following host/speaker. + */ + object BypassGlobalSpacePushCapForSoftDeviceFollow + extends FSParam[Boolean](name = "space_recs_bypass_global_pushcap_for_soft_follow", false) + + /* + * Param to bypass active listener predicate when target is device following host/speaker. + */ + object CheckActiveListenerPredicateForSoftDeviceFollow + extends FSParam[Boolean](name = "space_recs_check_active_listener_for_soft_follow", false) + + object SpreadControlRatioParam + extends FSBoundedParam[Double]( + name = "oon_spread_control_ratio", + default = 1000.0, + min = 0.0, + max = 100000.0 + ) + + object FavOverSendThresholdParam + extends FSBoundedParam[Double]( + name = "oon_spread_control_fav_over_send_threshold", + default = 0.14, + min = 0.0, + max = 1000.0 + ) + + object AuthorReportRateThresholdParam + extends FSBoundedParam[Double]( + name = "oon_spread_control_author_report_rate_threshold", + default = 7.4e-6, + min = 0.0, + max = 1000.0 + ) + + object AuthorDislikeRateThresholdParam + extends FSBoundedParam[Double]( + name = "oon_spread_control_author_dislike_rate_threshold", + default = 1.0, + min = 0.0, + max = 1000.0 + ) + + object MinTweetSendsThresholdParam + extends FSBoundedParam[Double]( + name = "oon_spread_control_min_tweet_sends_threshold", + default = 10000000000.0, + min = 0.0, + max = 10000000000.0 + ) + + object MinAuthorSendsThresholdParam + extends FSBoundedParam[Double]( + name = "oon_spread_control_min_author_sends_threshold", + default = 10000000000.0, + min = 0.0, + max = 10000000000.0 + ) + + /* + * Tweet Ntab-dislike predicate related params + */ + object TweetNtabDislikeCountThresholdParam + extends FSBoundedParam[Double]( + name = "oon_tweet_ntab_dislike_count_threshold", + default = 10000.0, + min = 0.0, + max = 10000.0 + ) + object TweetNtabDislikeRateThresholdParam + extends FSBoundedParam[Double]( + name = "oon_tweet_ntab_dislike_rate_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Param for tweet language feature name + */ + object TweetLanguageFeatureNameParam + extends FSParam[String]( + name = "language_tweet_language_feature_name", + default = "tweet.language.tweet.identified") + + /** + * Threshold for user inferred language filtering + */ + object UserInferredLanguageThresholdParam + extends FSBoundedParam[Double]( + name = "language_user_inferred_language_threshold", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + /** + * Threshold for user device language filtering + */ + object UserDeviceLanguageThresholdParam + extends FSBoundedParam[Double]( + name = "language_user_device_language_threshold", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + /** + * Param to enable/disable tweet language filter + */ + object EnableTweetLanguageFilter + extends FSParam[Boolean]( + name = "language_enable_tweet_language_filter", + default = false + ) + + /** + * Param to skip language filter for media tweets + */ + object SkipLanguageFilterForMediaTweets + extends FSParam[Boolean]( + name = "language_skip_language_filter_for_media_tweets", + default = false + ) + + /* + * Tweet Ntab-dislike predicate related params for MrTwistly + */ + object TweetNtabDislikeCountThresholdForMrTwistlyParam + extends FSBoundedParam[Double]( + name = "oon_tweet_ntab_dislike_count_threshold_for_mrtwistly", + default = 10000.0, + min = 0.0, + max = 10000.0 + ) + object TweetNtabDislikeRateThresholdForMrTwistlyParam + extends FSBoundedParam[Double]( + name = "oon_tweet_ntab_dislike_rate_threshold_for_mrtwistly", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + object TweetNtabDislikeCountBucketThresholdParam + extends FSBoundedParam[Double]( + name = "oon_tweet_ntab_dislike_count_bucket_threshold", + default = 10.0, + min = 0.0, + max = 10000.0 + ) + + /* + * Tweet engagement ratio predicate related params + */ + object TweetQTtoNtabClickRatioThresholdParam + extends FSBoundedParam[Double]( + name = "oon_tweet_engagement_filter_qt_to_ntabclick_ratio_threshold", + default = 0.0, + min = 0.0, + max = 100000.0 + ) + + /** + * Lower bound threshold to filter a tweet based on its reply to like ratio + */ + object TweetReplytoLikeRatioThresholdLowerBound + extends FSBoundedParam[Double]( + name = "oon_tweet_engagement_filter_reply_to_like_ratio_threshold_lower_bound", + default = Double.MaxValue, + min = 0.0, + max = Double.MaxValue + ) + + /** + * Upper bound threshold to filter a tweet based on its reply to like ratio + */ + object TweetReplytoLikeRatioThresholdUpperBound + extends FSBoundedParam[Double]( + name = "oon_tweet_engagement_filter_reply_to_like_ratio_threshold_upper_bound", + default = 0.0, + min = 0.0, + max = Double.MaxValue + ) + + /** + * Upper bound threshold to filter a tweet based on its reply to like ratio + */ + object TweetReplytoLikeRatioReplyCountThreshold + extends FSBoundedParam[Int]( + name = "oon_tweet_engagement_filter_reply_count_threshold", + default = Int.MaxValue, + min = 0, + max = Int.MaxValue + ) + + /* + * oonTweetLengthBasedPrerankingPredicate related params + */ + object OonTweetLengthPredicateUpdatedMediaLogic + extends FSParam[Boolean]( + name = "oon_quality_filter_tweet_length_updated_media_logic", + default = false + ) + + object OonTweetLengthPredicateUpdatedQuoteTweetLogic + extends FSParam[Boolean]( + name = "oon_quality_filter_tweet_length_updated_quote_tweet_logic", + default = false + ) + + object OonTweetLengthPredicateMoreStrictForUndefinedLanguages + extends FSParam[Boolean]( + name = "oon_quality_filter_tweet_length_more_strict_for_undefined_languages", + default = false + ) + + object EnablePrerankingTweetLengthPredicate + extends FSParam[Boolean]( + name = "oon_quality_filter_enable_preranking_filter", + default = false + ) + + /* + * LengthLanguageBasedOONTweetCandidatesQualityPredicate related params + */ + object SautOonWithMediaTweetLengthThresholdParam + extends FSBoundedParam[Double]( + name = "oon_quality_filter_tweet_length_threshold_for_saut_oon_with_media", + default = 0.0, + min = 0.0, + max = 70.0 + ) + object NonSautOonWithMediaTweetLengthThresholdParam + extends FSBoundedParam[Double]( + name = "oon_quality_filter_tweet_length_threshold_for_non_saut_oon_with_media", + default = 0.0, + min = 0.0, + max = 70.0 + ) + object SautOonWithoutMediaTweetLengthThresholdParam + extends FSBoundedParam[Double]( + name = "oon_quality_filter_tweet_length_threshold_for_saut_oon_without_media", + default = 0.0, + min = 0.0, + max = 70.0 + ) + object NonSautOonWithoutMediaTweetLengthThresholdParam + extends FSBoundedParam[Double]( + name = "oon_quality_filter_tweet_length_threshold_for_non_saut_oon_without_media", + default = 0.0, + min = 0.0, + max = 70.0 + ) + + object ArgfOonWithMediaTweetWordLengthThresholdParam + extends FSBoundedParam[Double]( + name = "oon_quality_filter_tweet_word_length_threshold_for_argf_oon_with_media", + default = 0.0, + min = 0.0, + max = 18.0 + ) + object EsfthOonWithMediaTweetWordLengthThresholdParam + extends FSBoundedParam[Double]( + name = "oon_quality_filter_tweet_word_length_threshold_for_esfth_oon_with_media", + default = 0.0, + min = 0.0, + max = 10.0 + ) + + /** + * Param to enable/disable sentiment feature hydration + */ + object EnableMrTweetSentimentFeatureHydrationFS + extends FSParam[Boolean]( + name = "feature_hydration_enable_mr_tweet_sentiment_feature", + default = false + ) + + /** + * Param to enable/disable feature map scribing for staging test log + */ + object EnableMrScribingMLFeaturesAsFeatureMapForStaging + extends FSParam[Boolean]( + name = "frigate_pushservice_enable_scribing_ml_features_as_featuremap_for_staging", + default = false + ) + + /** + * Param to enable timeline health signal hydration + * */ + object EnableTimelineHealthSignalHydration + extends FSParam[Boolean]( + name = "timeline_health_signal_hydration", + default = false + ) + + /** + * Param to enable timeline health signal hydration for model training + * */ + object EnableTimelineHealthSignalHydrationForModelTraining + extends FSParam[Boolean]( + name = "timeline_health_signal_hydration_for_model_training", + default = false + ) + + /** + * Param to enable/disable mr user social context agg feature hydration + */ + object EnableMrUserSocialContextAggregateFeatureHydration + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_mr_user_social_context_agg_feature", + default = true + ) + + /** + * Param to enable/disable mr user semantic core agg feature hydration + */ + object EnableMrUserSemanticCoreAggregateFeatureHydration + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_mr_user_semantic_core_agg_feature", + default = true + ) + + /** + * Param to enable/disable mr user candidate sparse agg feature hydration + */ + object EnableMrUserCandidateSparseOfflineAggregateFeatureHydration + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_mr_user_candidate_sparse_agg_feature", + default = true + ) + + /** + * Param to enable/disable mr user candidate agg feature hydration + */ + object EnableMrUserCandidateOfflineAggregateFeatureHydration + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_mr_user_candidate_agg_feature", + default = true + ) + + /** + * Param to enable/disable mr user candidate compact agg feature hydration + */ + object EnableMrUserCandidateOfflineCompactAggregateFeatureHydration + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_mr_user_candidate_compact_agg_feature", + default = false + ) + + /** + * Param to enable/disable mr real graph user-author/social-context feature hydration + */ + object EnableRealGraphUserAuthorAndSocialContxtFeatureHydration + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_real_graph_user_social_feature", + default = true + ) + + /** + * Param to enable/disable mr user author agg feature hydration + */ + object EnableMrUserAuthorOfflineAggregateFeatureHydration + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_mr_user_author_agg_feature", + default = true + ) + + /** + * Param to enable/disable mr user author compact agg feature hydration + */ + object EnableMrUserAuthorOfflineCompactAggregateFeatureHydration + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_mr_user_author_compact_agg_feature", + default = false + ) + + /** + * Param to enable/disable mr user compact agg feature hydration + */ + object EnableMrUserOfflineCompactAggregateFeatureHydration + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_mr_user_compact_agg_feature", + default = false + ) + + /** + * Param to enable/disable mr user simcluster agg feature hydration + */ + object EnableMrUserSimcluster2020AggregateFeatureHydration + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_mr_user_simcluster_agg_feature", + default = true + ) + + /** + * Param to enable/disable mr user agg feature hydration + */ + object EnableMrUserOfflineAggregateFeatureHydration + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_mr_user_agg_feature", + default = true + ) + + /** + * Param to enable/disable topic engagement RTA in the ranking model + */ + object EnableTopicEngagementRealTimeAggregatesFS + extends FSParam[Boolean]( + "feature_hydration_enable_htl_topic_engagement_real_time_agg_feature", + false) + + /* + * Param to enable mr user semantic core feature hydration for heavy ranker + * */ + object EnableMrUserSemanticCoreFeatureForExpt + extends FSParam[Boolean]( + name = "frigate_push_modeling_hydrate_mr_user_semantic_core", + default = false) + + /** + * Param to enable hydrating user duration since last visit features + */ + object EnableHydratingUserDurationSinceLastVisitFeatures + extends FSParam[Boolean]( + name = "feature_hydration_user_duration_since_last_visit", + default = false) + + /** + Param to enable/disable user-topic aggregates in the ranking model + */ + object EnableUserTopicAggregatesFS + extends FSParam[Boolean]("feature_hydration_enable_htl_topic_user_agg_feature", false) + + /* + * PNegMultimodalPredicate related params + */ + object EnablePNegMultimodalPredicateParam + extends FSParam[Boolean]( + name = "pneg_multimodal_filter_enable_param", + default = false + ) + object PNegMultimodalPredicateModelThresholdParam + extends FSBoundedParam[Double]( + name = "pneg_multimodal_filter_model_threshold_param", + default = 1.0, + min = 0.0, + max = 1.0 + ) + object PNegMultimodalPredicateBucketThresholdParam + extends FSBoundedParam[Double]( + name = "pneg_multimodal_filter_bucket_threshold_param", + default = 0.4, + min = 0.0, + max = 1.0 + ) + + /* + * NegativeKeywordsPredicate related params + */ + object EnableNegativeKeywordsPredicateParam + extends FSParam[Boolean]( + name = "negative_keywords_filter_enable_param", + default = false + ) + object NegativeKeywordsPredicateDenylist + extends FSParam[Seq[String]]( + name = "negative_keywords_filter_denylist", + default = Seq.empty[String] + ) + /* + * LightRanking related params + */ + object EnableLightRankingParam + extends FSParam[Boolean]( + name = "light_ranking_enable_param", + default = false + ) + object LightRankingNumberOfCandidatesParam + extends FSBoundedParam[Int]( + name = "light_ranking_number_of_candidates_param", + default = 100, + min = 0, + max = 1000 + ) + object LightRankingModelTypeParam + extends FSParam[String]( + name = "light_ranking_model_type_param", + default = "WeightedOpenOrNtabClickProbability_Q4_2021_13172_Mr_Light_Ranker_Dbv2_Top3") + object EnableRandomBaselineLightRankingParam + extends FSParam[Boolean]( + name = "light_ranking_random_baseline_enable_param", + default = false + ) + + object LightRankingScribeCandidatesDownSamplingParam + extends FSBoundedParam[Double]( + name = "light_ranking_scribe_candidates_down_sampling_param", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /* + * Quality Upranking related params + */ + object EnableProducersQualityBoostingForHeavyRankingParam + extends FSParam[Boolean]( + name = "quality_upranking_enable_producers_quality_boosting_for_heavy_ranking_param", + default = false + ) + + object QualityUprankingBoostForHighQualityProducersParam + extends FSBoundedParam[Double]( + name = "quality_upranking_boost_for_high_quality_producers_param", + default = 1.0, + min = 0.0, + max = 10000.0 + ) + + object QualityUprankingDownboostForLowQualityProducersParam + extends FSBoundedParam[Double]( + name = "quality_upranking_downboost_for_low_quality_producers_param", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + object EnableQualityUprankingForHeavyRankingParam + extends FSParam[Boolean]( + name = "quality_upranking_enable_for_heavy_ranking_param", + default = false + ) + object QualityUprankingModelTypeParam + extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( + name = "quality_upranking_model_id", + default = "Q4_2022_Mr_Bqml_Quality_Model_wALL" + ) + object QualityUprankingTransformTypeParam + extends FSEnumParam[MrQualityUprankingTransformTypeEnum.type]( + name = "quality_upranking_transform_id", + default = MrQualityUprankingTransformTypeEnum.Sigmoid, + enum = MrQualityUprankingTransformTypeEnum + ) + + object QualityUprankingBoostForHeavyRankingParam + extends FSBoundedParam[Double]( + name = "quality_upranking_boost_for_heavy_ranking_param", + default = 1.0, + min = -10.0, + max = 10.0 + ) + object QualityUprankingSigmoidBiasForHeavyRankingParam + extends FSBoundedParam[Double]( + name = "quality_upranking_sigmoid_bias_for_heavy_ranking_param", + default = 0.0, + min = -10.0, + max = 10.0 + ) + object QualityUprankingSigmoidWeightForHeavyRankingParam + extends FSBoundedParam[Double]( + name = "quality_upranking_sigmoid_weight_for_heavy_ranking_param", + default = 1.0, + min = -10.0, + max = 10.0 + ) + object QualityUprankingLinearBarForHeavyRankingParam + extends FSBoundedParam[Double]( + name = "quality_upranking_linear_bar_for_heavy_ranking_param", + default = 1.0, + min = 0.0, + max = 10.0 + ) + object EnableQualityUprankingCrtScoreStatsForHeavyRankingParam + extends FSParam[Boolean]( + name = "quality_upranking_enable_crt_score_stats_for_heavy_ranking_param", + default = false + ) + /* + * BQML Health Model related params + */ + object EnableBqmlHealthModelPredicateParam + extends FSParam[Boolean]( + name = "bqml_health_model_filter_enable_param", + default = false + ) + + object EnableBqmlHealthModelPredictionForInNetworkCandidatesParam + extends FSParam[Boolean]( + name = "bqml_health_model_enable_prediction_for_in_network_candidates_param", + default = false + ) + + object BqmlHealthModelTypeParam + extends FSParam[HealthNsfwModel.ModelNameType]( + name = "bqml_health_model_id", + default = HealthNsfwModel.Q2_2022_Mr_Bqml_Health_Model_NsfwV0 + ) + object BqmlHealthModelPredicateFilterThresholdParam + extends FSBoundedParam[Double]( + name = "bqml_health_model_filter_threshold_param", + default = 1.0, + min = 0.0, + max = 1.0 + ) + object BqmlHealthModelPredicateBucketThresholdParam + extends FSBoundedParam[Double]( + name = "bqml_health_model_bucket_threshold_param", + default = 0.005, + min = 0.0, + max = 1.0 + ) + + object EnableBqmlHealthModelScoreHistogramParam + extends FSParam[Boolean]( + name = "bqml_health_model_score_histogram_enable_param", + default = false + ) + + /* + * BQML Quality Model related params + */ + object EnableBqmlQualityModelPredicateParam + extends FSParam[Boolean]( + name = "bqml_quality_model_filter_enable_param", + default = false + ) + object EnableBqmlQualityModelScoreHistogramParam + extends FSParam[Boolean]( + name = "bqml_quality_model_score_histogram_enable_param", + default = false + ) + object BqmlQualityModelTypeParam + extends FSParam[WeightedOpenOrNtabClickModel.ModelNameType]( + name = "bqml_quality_model_id", + default = "Q1_2022_13562_Mr_Bqml_Quality_Model_V2" + ) + + /** + * Param to specify which quality models to use to get the scores for determining + * whether to bucket a user for the DDG + */ + object BqmlQualityModelBucketModelIdListParam + extends FSParam[Seq[WeightedOpenOrNtabClickModel.ModelNameType]]( + name = "bqml_quality_model_bucket_model_id_list", + default = Seq( + "Q1_2022_13562_Mr_Bqml_Quality_Model_V2", + "Q2_2022_DDG14146_Mr_Personalised_BQML_Quality_Model", + "Q2_2022_DDG14146_Mr_NonPersonalised_BQML_Quality_Model" + ) + ) + + object BqmlQualityModelPredicateThresholdParam + extends FSBoundedParam[Double]( + name = "bqml_quality_model_filter_threshold_param", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Param to specify the threshold to determine if a user’s quality score is high enough to enter the experiment. + */ + object BqmlQualityModelBucketThresholdListParam + extends FSParam[Seq[Double]]( + name = "bqml_quality_model_bucket_threshold_list", + default = Seq(0.7, 0.7, 0.7) + ) + + /* + * TweetAuthorAggregates related params + */ + object EnableTweetAuthorAggregatesFeatureHydrationParam + extends FSParam[Boolean]( + name = "tweet_author_aggregates_feature_hydration_enable_param", + default = false + ) + + /** + * Param to determine if we should include the relevancy score of candidates in the Ibis payload + */ + object IncludeRelevanceScoreInIbis2Payload + extends FSParam[Boolean]( + name = "relevance_score_include_in_ibis2_payload", + default = false + ) + + /** + * Param to specify supervised model to predict score by sending the notification + */ + object BigFilteringSupervisedSendingModelParam + extends FSParam[BigFilteringSupervisedModel.ModelNameType]( + name = "ltv_filtering_bigfiltering_supervised_sending_model_param", + default = BigFilteringSupervisedModel.V0_0_BigFiltering_Supervised_Sending_Model + ) + + /** + * Param to specify supervised model to predict score by not sending the notification + */ + object BigFilteringSupervisedWithoutSendingModelParam + extends FSParam[BigFilteringSupervisedModel.ModelNameType]( + name = "ltv_filtering_bigfiltering_supervised_without_sending_model_param", + default = BigFilteringSupervisedModel.V0_0_BigFiltering_Supervised_Without_Sending_Model + ) + + /** + * Param to specify RL model to predict score by sending the notification + */ + object BigFilteringRLSendingModelParam + extends FSParam[BigFilteringSupervisedModel.ModelNameType]( + name = "ltv_filtering_bigfiltering_rl_sending_model_param", + default = BigFilteringRLModel.V0_0_BigFiltering_Rl_Sending_Model + ) + + /** + * Param to specify RL model to predict score by not sending the notification + */ + object BigFilteringRLWithoutSendingModelParam + extends FSParam[BigFilteringSupervisedModel.ModelNameType]( + name = "ltv_filtering_bigfiltering_rl_without_sending_model_param", + default = BigFilteringRLModel.V0_0_BigFiltering_Rl_Without_Sending_Model + ) + + /** + * Param to specify the threshold (send notification if score >= threshold) + */ + object BigFilteringThresholdParam + extends FSBoundedParam[Double]( + name = "ltv_filtering_bigfiltering_threshold_param", + default = 0.0, + min = Double.MinValue, + max = Double.MaxValue + ) + + /** + * Param to specify normalization used for BigFiltering + */ + object BigFilteringNormalizationTypeIdParam + extends FSEnumParam[BigFilteringNormalizationEnum.type]( + name = "ltv_filtering_bigfiltering_normalization_type_id", + default = BigFilteringNormalizationEnum.NormalizationDisabled, + enum = BigFilteringNormalizationEnum + ) + + /** + * Param to specify histograms of model scores in BigFiltering + */ + object BigFilteringEnableHistogramsParam + extends FSParam[Boolean]( + name = "ltv_filtering_bigfiltering_enable_histograms_param", + default = false + ) + + /* + * Param to enable sending requests to Ins Sender + */ + object EnableInsSender extends FSParam[Boolean](name = "ins_enable_dark_traffic", default = false) + + /** + * Param to specify the range of relevance scores for MagicFanout types. + */ + object MagicFanoutRelevanceScoreRange + extends FSParam[Seq[Double]]( + name = "relevance_score_mf_range", + default = Seq(0.75, 1.0) + ) + + /** + * Param to specify the range of relevance scores for MR types. + */ + object MagicRecsRelevanceScoreRange + extends FSParam[Seq[Double]]( + name = "relevance_score_mr_range", + default = Seq(0.25, 0.5) + ) + + /** + * Param to enable backfilling OON candidates if number of F1 candidates is greater than a threshold K. + */ + object EnableOONBackfillBasedOnF1Candidates + extends FSParam[Boolean](name = "oon_enable_backfill_based_on_f1", default = false) + + /** + * Threshold for the minimum number of F1 candidates required to enable backfill of OON candidates. + */ + object NumberOfF1CandidatesThresholdForOONBackfill + extends FSBoundedParam[Int]( + name = "oon_enable_backfill_f1_threshold", + min = 0, + default = 5000, + max = 5000) + + /** + * Event ID allowlist to skip account country predicate + */ + object MagicFanoutEventAllowlistToSkipAccountCountryPredicate + extends FSParam[Seq[Long]]( + name = "magicfanout_event_allowlist_skip_account_country_predicate", + default = Seq.empty[Long] + ) + + /** + * MagicFanout Event Semantic Core Domain Ids + */ + object ListOfEventSemanticCoreDomainIds + extends FSParam[Seq[Long]]( + name = "magicfanout_automated_events_semantic_core_domain_ids", + default = Seq()) + + /** + * Adhoc id for detailed rank flow stats + */ + object ListOfAdhocIdsForStatsTracking + extends FSParam[Set[Long]]( + name = "stats_enable_detailed_stats_tracking_ids", + default = Set.empty[Long] + ) + + object EnableGenericCRTBasedFatiguePredicate + extends FSParam[Boolean]( + name = "seelessoften_enable_generic_crt_based_fatigue_predicate", + default = false) + + /** + * Param to enable copy features such as Emojis and Target Name + */ + object EnableCopyFeaturesForF1 + extends FSParam[Boolean](name = "mr_copy_enable_features_f1", default = false) + + /** + * Param to enable copy features such as Emojis and Target Name + */ + object EnableCopyFeaturesForOon + extends FSParam[Boolean](name = "mr_copy_enable_features_oon", default = false) + + /** + * Param to enable Emoji in F1 Copy + */ + object EnableEmojiInF1Copy + extends FSParam[Boolean](name = "mr_copy_enable_f1_emoji", default = false) + + /** + * Param to enable Target in F1 Copy + */ + object EnableTargetInF1Copy + extends FSParam[Boolean](name = "mr_copy_enable_f1_target", default = false) + + /** + * Param to enable Emoji in OON Copy + */ + object EnableEmojiInOonCopy + extends FSParam[Boolean](name = "mr_copy_enable_oon_emoji", default = false) + + /** + * Param to enable Target in OON Copy + */ + object EnableTargetInOonCopy + extends FSParam[Boolean](name = "mr_copy_enable_oon_target", default = false) + + /** + * Param to enable split fatigue for Target and Emoji copy for OON and F1 + */ + object EnableTargetAndEmojiSplitFatigue + extends FSParam[Boolean](name = "mr_copy_enable_target_emoji_split_fatigue", default = false) + + /** + * Param to enable experimenting string on the body + */ + object EnableF1CopyBody extends FSParam[Boolean](name = "mr_copy_f1_enable_body", default = false) + + object EnableOONCopyBody + extends FSParam[Boolean](name = "mr_copy_oon_enable_body", default = false) + + object EnableIosCopyBodyTruncate + extends FSParam[Boolean](name = "mr_copy_enable_body_truncate", default = false) + + object EnableNsfwCopy extends FSParam[Boolean](name = "mr_copy_enable_nsfw", default = false) + + /** + * Param to determine F1 candidate nsfw score threshold + */ + object NsfwScoreThresholdForF1Copy + extends FSBoundedParam[Double]( + name = "mr_copy_nsfw_threshold_f1", + default = 0.3, + min = 0.0, + max = 1.0 + ) + + /** + * Param to determine OON candidate nsfw score threshold + */ + object NsfwScoreThresholdForOONCopy + extends FSBoundedParam[Double]( + name = "mr_copy_nsfw_threshold_oon", + default = 0.2, + min = 0.0, + max = 1.0 + ) + + /** + * Param to determine the lookback duration when searching for prev copy features. + */ + object CopyFeaturesHistoryLookbackDuration + extends FSBoundedParam[Duration]( + name = "mr_copy_history_lookback_duration_in_days", + default = 30.days, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to determine the F1 emoji copy fatigue in # of hours. + */ + object F1EmojiCopyFatigueDuration + extends FSBoundedParam[Duration]( + name = "mr_copy_f1_emoji_copy_fatigue_in_hours", + default = 24.hours, + min = 0.hours, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to determine the F1 target copy fatigue in # of hours. + */ + object F1TargetCopyFatigueDuration + extends FSBoundedParam[Duration]( + name = "mr_copy_f1_target_copy_fatigue_in_hours", + default = 24.hours, + min = 0.hours, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to determine the OON emoji copy fatigue in # of hours. + */ + object OonEmojiCopyFatigueDuration + extends FSBoundedParam[Duration]( + name = "mr_copy_oon_emoji_copy_fatigue_in_hours", + default = 24.hours, + min = 0.hours, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to determine the OON target copy fatigue in # of hours. + */ + object OonTargetCopyFatigueDuration + extends FSBoundedParam[Duration]( + name = "mr_copy_oon_target_copy_fatigue_in_hours", + default = 24.hours, + min = 0.hours, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to turn on/off home timeline based fatigue rule, where once last home timeline visit + * is larger than the specified will evalute to not fatigue + */ + object EnableHTLBasedFatigueBasicRule + extends FSParam[Boolean]( + name = "mr_copy_enable_htl_based_fatigue_basic_rule", + default = false) + + /** + * Param to determine f1 emoji copy fatigue in # of pushes + */ + object F1EmojiCopyNumOfPushesFatigue + extends FSBoundedParam[Int]( + name = "mr_copy_f1_emoji_copy_number_of_pushes_fatigue", + default = 0, + min = 0, + max = 200 + ) + + /** + * Param to determine oon emoji copy fatigue in # of pushes + */ + object OonEmojiCopyNumOfPushesFatigue + extends FSBoundedParam[Int]( + name = "mr_copy_oon_emoji_copy_number_of_pushes_fatigue", + default = 0, + min = 0, + max = 200 + ) + + /** + * If user haven't visited home timeline for certain duration, we will + * exempt user from feature copy fatigue. This param is used to control + * how long it is before we enter exemption. + */ + object MinFatigueDurationSinceLastHTLVisit + extends FSBoundedParam[Duration]( + name = "mr_copy_min_duration_since_last_htl_visit_hours", + default = Duration.Top, + min = 0.hour, + max = Duration.Top, + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * If a user haven't visit home timeline very long, the user will return + * to fatigue state under the home timeline based fatigue rule. There will + * only be a window, where the user is out of fatigue state under the rule. + * This param control the length of the non fatigue period. + */ + object LastHTLVisitBasedNonFatigueWindow + extends FSBoundedParam[Duration]( + name = "mr_copy_last_htl_visit_based_non_fatigue_window_hours", + default = 48.hours, + min = 0.hour, + max = Duration.Top, + ) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + object EnableOONCBasedCopy + extends FSParam[Boolean]( + name = "mr_copy_enable_oonc_based_copy", + default = false + ) + + object HighOONCThresholdForCopy + extends FSBoundedParam[Double]( + name = "mr_copy_high_oonc_threshold_for_copy", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + object LowOONCThresholdForCopy + extends FSBoundedParam[Double]( + name = "mr_copy_low_oonc_threshold_for_copy", + default = 0.0, + min = 0.0, + max = 1.0 + ) + + object EnableTweetTranslation + extends FSParam[Boolean](name = "tweet_translation_enable", default = false) + + object TripTweetCandidateReturnEnable + extends FSParam[Boolean](name = "trip_tweet_candidate_enable", default = false) + + object TripTweetCandidateSourceIds + extends FSParam[Seq[String]]( + name = "trip_tweet_candidate_source_ids", + default = Seq("TOP_GEO_V3")) + + object TripTweetMaxTotalCandidates + extends FSBoundedParam[Int]( + name = "trip_tweet_max_total_candidates", + default = 500, + min = 10, + max = 1000) + + object EnableEmptyBody + extends FSParam[Boolean](name = "push_presentation_enable_empty_body", default = false) + + object EnableSocialContextForRetweet + extends FSParam[Boolean](name = "push_presentation_social_context_retweet", default = false) + + /** + * Param to enable/disable simcluster feature hydration + */ + object EnableMrTweetSimClusterFeatureHydrationFS + extends FSParam[Boolean]( + name = "feature_hydration_enable_mr_tweet_simcluster_feature", + default = false + ) + + /** + * Param to disable OON candidates based on tweetAuthor + */ + object DisableOutNetworkTweetCandidatesFS + extends FSParam[Boolean](name = "oon_filtering_disable_oon_candidates", default = false) + + /** + * Param to enable Local Viral Tweets + */ + object EnableLocalViralTweets + extends FSParam[Boolean](name = "local_viral_tweets_enable", default = true) + + /** + * Param to enable Explore Video Tweets + */ + object EnableExploreVideoTweets + extends FSParam[Boolean](name = "explore_video_tweets_enable", default = false) + + /** + * Param to enable List Recommendations + */ + object EnableListRecommendations + extends FSParam[Boolean](name = "list_recommendations_enable", default = false) + + /** + * Param to enable IDS List Recommendations + */ + object EnableIDSListRecommendations + extends FSParam[Boolean](name = "list_recommendations_ids_enable", default = false) + + /** + * Param to enable PopGeo List Recommendations + */ + object EnablePopGeoListRecommendations + extends FSParam[Boolean](name = "list_recommendations_pop_geo_enable", default = false) + + /** + * Param to control the inverter for fatigue between consecutive ListRecommendations + */ + object ListRecommendationsPushInterval + extends FSBoundedParam[Duration]( + name = "list_recommendations_interval_days", + default = 24.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromDays + } + + /** + * Param to control the granularity of GeoHash for ListRecommendations + */ + object ListRecommendationsGeoHashLength + extends FSBoundedParam[Int]( + name = "list_recommendations_geo_hash_length", + default = 5, + min = 3, + max = 5) + + /** + * Param to control maximum number of ListRecommendation pushes to receive in an interval + */ + object MaxListRecommendationsPushGivenInterval + extends FSBoundedParam[Int]( + name = "list_recommendations_push_given_interval", + default = 1, + min = 0, + max = 10 + ) + + /** + * Param to control the subscriber count for list recommendation + */ + object ListRecommendationsSubscriberCount + extends FSBoundedParam[Int]( + name = "list_recommendations_subscriber_count", + default = 0, + min = 0, + max = Integer.MAX_VALUE) + + /** + * Param to define dynamic inline action types for web notifications (both desktop web + mobile web) + */ + object LocalViralTweetsBucket + extends FSParam[String]( + name = "local_viral_tweets_bucket", + default = "high", + ) + + /** + * List of CrTags to disable + */ + object OONCandidatesDisabledCrTagParam + extends FSParam[Seq[String]]( + name = "oon_enable_oon_candidates_disabled_crtag", + default = Seq.empty[String] + ) + + /** + * List of Crt groups to disable + */ + object OONCandidatesDisabledCrtGroupParam + extends FSEnumSeqParam[CrtGroupEnum.type]( + name = "oon_enable_oon_candidates_disabled_crt_group_ids", + default = Seq.empty[CrtGroupEnum.Value], + enum = CrtGroupEnum + ) + + /** + * Param to enable launching video tweets in the Immersive Explore timeline + */ + object EnableLaunchVideosInImmersiveExplore + extends FSParam[Boolean](name = "launch_videos_in_immersive_explore", default = false) + + /** + * Param to enable Ntab Entries for Sports Event Notifications + */ + object EnableNTabEntriesForSportsEventNotifications + extends FSParam[Boolean]( + name = "magicfanout_sports_event_enable_ntab_entries", + default = false) + + /** + * Param to enable Ntab Facepiles for teams in Sport Notifs + */ + object EnableNTabFacePileForSportsEventNotifications + extends FSParam[Boolean]( + name = "magicfanout_sports_event_enable_ntab_facepiles", + default = false) + + /** + * Param to enable Ntab Override for Sports Event Notifications + */ + object EnableNTabOverrideForSportsEventNotifications + extends FSParam[Boolean]( + name = "magicfanout_sports_event_enable_ntab_override", + default = false) + + /** + * Param to control the interval for MF Product Launch Notifs + */ + object ProductLaunchPushIntervalInHours + extends FSBoundedParam[Duration]( + name = "product_launch_fatigue_push_interval_in_hours", + default = 24.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the maximum number of MF Product Launch Notifs in a period of time + */ + object ProductLaunchMaxNumberOfPushesInInterval + extends FSBoundedParam[Int]( + name = "product_launch_fatigue_max_pushes_in_interval", + default = 1, + min = 0, + max = 10) + + /** + * Param to control the minInterval for fatigue between consecutive MF Product Launch Notifs + */ + object ProductLaunchMinIntervalFatigue + extends FSBoundedParam[Duration]( + name = "product_launch_fatigue_min_interval_consecutive_pushes_in_hours", + default = 24.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the interval for MF New Creator Notifs + */ + object NewCreatorPushIntervalInHours + extends FSBoundedParam[Duration]( + name = "new_creator_fatigue_push_interval_in_hours", + default = 24.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the maximum number of MF New Creator Notifs in a period of time + */ + object NewCreatorPushMaxNumberOfPushesInInterval + extends FSBoundedParam[Int]( + name = "new_creator_fatigue_max_pushes_in_interval", + default = 1, + min = 0, + max = 10) + + /** + * Param to control the minInterval for fatigue between consecutive MF New Creator Notifs + */ + object NewCreatorPushMinIntervalFatigue + extends FSBoundedParam[Duration]( + name = "new_creator_fatigue_min_interval_consecutive_pushes_in_hours", + default = 24.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the interval for MF New Creator Notifs + */ + object CreatorSubscriptionPushIntervalInHours + extends FSBoundedParam[Duration]( + name = "creator_subscription_fatigue_push_interval_in_hours", + default = 24.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to control the maximum number of MF New Creator Notifs in a period of time + */ + object CreatorSubscriptionPushMaxNumberOfPushesInInterval + extends FSBoundedParam[Int]( + name = "creator_subscription_fatigue_max_pushes_in_interval", + default = 1, + min = 0, + max = 10) + + /** + * Param to control the minInterval for fatigue between consecutive MF New Creator Notifs + */ + object CreatorSubscriptionPushhMinIntervalFatigue + extends FSBoundedParam[Duration]( + name = "creator_subscription_fatigue_min_interval_consecutive_pushes_in_hours", + default = 24.hours, + min = Duration.Bottom, + max = Duration.Top) + with HasDurationConversion { + override val durationConversion = DurationConversion.FromHours + } + + /** + * Param to define the landing page deeplink of product launch notifications + */ + object ProductLaunchLandingPageDeepLink + extends FSParam[String]( + name = "product_launch_landing_page_deeplink", + default = "" + ) + + /** + * Param to define the tap through of product launch notifications + */ + object ProductLaunchTapThrough + extends FSParam[String]( + name = "product_launch_tap_through", + default = "" + ) + + /** + * Param to skip checking isTargetBlueVerified + */ + object DisableIsTargetBlueVerifiedPredicate + extends FSParam[Boolean]( + name = "product_launch_disable_is_target_blue_verified_predicate", + default = false + ) + + /** + * Param to enable Ntab Entries for Sports Event Notifications + */ + object EnableNTabEntriesForProductLaunchNotifications + extends FSParam[Boolean](name = "product_launch_enable_ntab_entry", default = true) + + /** + * Param to skip checking isTargetLegacyVerified + */ + object DisableIsTargetLegacyVerifiedPredicate + extends FSParam[Boolean]( + name = "product_launch_disable_is_target_legacy_verified_predicate", + default = false + ) + + /** + * Param to enable checking isTargetSuperFollowCreator + */ + object EnableIsTargetSuperFollowCreatorPredicate + extends FSParam[Boolean]( + name = "product_launch_is_target_super_follow_creator_predicate_enabled", + default = false + ) + + /** + * Param to enable Spammy Tweet filter + */ + object EnableSpammyTweetFilter + extends FSParam[Boolean]( + name = "health_signal_store_enable_spammy_tweet_filter", + default = false) + + /** + * Param to enable Push to Home Android + */ + object EnableTweetPushToHomeAndroid + extends FSParam[Boolean](name = "push_to_home_tweet_recs_android", default = false) + + /** + * Param to enable Push to Home iOS + */ + object EnableTweetPushToHomeiOS + extends FSParam[Boolean](name = "push_to_home_tweet_recs_iOS", default = false) + + /** + * Param to set Spammy Tweet score threshold for OON candidates + */ + object SpammyTweetOonThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_spammy_tweet_oon_threshold", + default = 1.1, + min = 0.0, + max = 1.1 + ) + + object NumFollowerThresholdForHealthAndQualityFilters + extends FSBoundedParam[Double]( + name = "health_signal_store_num_follower_threshold_for_health_and_quality_filters", + default = 10000000000.0, + min = 0.0, + max = 10000000000.0 + ) + + object NumFollowerThresholdForHealthAndQualityFiltersPreranking + extends FSBoundedParam[Double]( + name = + "health_signal_store_num_follower_threshold_for_health_and_quality_filters_preranking", + default = 10000000.0, + min = 0.0, + max = 10000000000.0 + ) + + /** + * Param to set Spammy Tweet score threshold for IN candidates + */ + object SpammyTweetInThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_spammy_tweet_in_threshold", + default = 1.1, + min = 0.0, + max = 1.1 + ) + + /** + * Param to control bucketing for the Spammy Tweet score + */ + object SpammyTweetBucketingThreshold + extends FSBoundedParam[Double]( + name = "health_signal_store_spammy_tweet_bucketing_threshold", + default = 1.0, + min = 0.0, + max = 1.0 + ) + + /** + * Param to specify the maximum number of Explore Video Tweets to request + */ + object MaxExploreVideoTweets + extends FSBoundedParam[Int]( + name = "explore_video_tweets_max_candidates", + default = 100, + min = 0, + max = 500 + ) + + /** + * Param to enable social context feature set + */ + object EnableBoundedFeatureSetForSocialContext + extends FSParam[Boolean]( + name = "feature_hydration_user_social_context_bounded_feature_set_enable", + default = true) + + /** + * Param to enable stp user social context feature set + */ + object EnableStpBoundedFeatureSetForUserSocialContext + extends FSParam[Boolean]( + name = "feature_hydration_stp_social_context_bounded_feature_set_enable", + default = true) + + /** + * Param to enable core user history social context feature set + */ + object EnableCoreUserHistoryBoundedFeatureSetForSocialContext + extends FSParam[Boolean]( + name = "feature_hydration_core_user_history_social_context_bounded_feature_set_enable", + default = true) + + /** + * Param to enable skipping post-ranking filters + */ + object SkipPostRankingFilters + extends FSParam[Boolean]( + name = "frigate_push_modeling_skip_post_ranking_filters", + default = false) + + object MagicFanoutSimClusterDotProductNonHeavyUserThreshold + extends FSBoundedParam[Double]( + name = "frigate_push_magicfanout_simcluster_non_heavy_user_dot_product_threshold", + default = 0.0, + min = 0.0, + max = 100.0 + ) + + object MagicFanoutSimClusterDotProductHeavyUserThreshold + extends FSBoundedParam[Double]( + name = "frigate_push_magicfanout_simcluster_heavy_user_dot_product_threshold", + default = 10.0, + min = 0.0, + max = 100.0 + ) + + object EnableReducedFatigueRulesForSeeLessOften + extends FSParam[Boolean]( + name = "seelessoften_enable_reduced_fatigue", + default = false + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.scala new file mode 100644 index 0000000000..96167c1340 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushFeatureSwitches.scala @@ -0,0 +1,751 @@ +package com.twitter.frigate.pushservice.params + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.util.{FeatureSwitchParams => Common} +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => Pushservice} +import com.twitter.logging.Logger +import com.twitter.servo.decider.DeciderGateBuilder +import com.twitter.timelines.configapi.BaseConfigBuilder +import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil +import com.twitter.timelines.configapi.decider.DeciderUtils + +case class PushFeatureSwitches( + deciderGateBuilder: DeciderGateBuilder, + statsReceiver: StatsReceiver) { + + private[this] val logger = Logger(classOf[PushFeatureSwitches]) + private[this] val stat = statsReceiver.scope("PushFeatureSwitches") + + private val booleanDeciderOverrides = DeciderUtils.getBooleanDeciderOverrides( + deciderGateBuilder, + PushParams.DisableAllRelevanceParam, + PushParams.DisableHeavyRankingParam, + PushParams.RestrictLightRankingParam, + PushParams.UTEGTweetCandidateSourceParam, + PushParams.EnableWritesToNotificationServiceParam, + PushParams.EnableWritesToNotificationServiceForAllEmployeesParam, + PushParams.EnableWritesToNotificationServiceForEveryoneParam, + PushParams.EnablePromptFeedbackFatigueResponseNoPredicate, + PushParams.EarlyBirdSCBasedCandidatesParam, + PushParams.MRTweetFavRecsParam, + PushParams.MRTweetRetweetRecsParam, + PushParams.EnablePushSendEventBus, + PushParams.DisableMlInFilteringParam, + PushParams.DownSampleLightRankingScribeCandidatesParam, + PushParams.EnableMrRequestScribing, + PushParams.EnableHighQualityCandidateScoresScribing, + PushParams.EnablePnegMultimodalPredictionForF1Tweets, + PushParams.EnableScribeOonFavScoreForF1Tweets, + PushParams.EnableMrUserSemanticCoreFeaturesHydration, + PushParams.EnableMrUserSemanticCoreNoZeroFeaturesHydration, + PushParams.EnableHtlOfflineUserAggregatesExtendedHydration, + PushParams.EnableNerErgFeatureHydration, + PushParams.EnableDaysSinceRecentResurrectionFeatureHydration, + PushParams.EnableUserPastAggregatesFeatureHydration, + PushParams.EnableMrUserSimclusterV2020FeaturesHydration, + PushParams.EnableMrUserSimclusterV2020NoZeroFeaturesHydration, + PushParams.EnableTopicEngagementRealTimeAggregatesFeatureHydration, + PushParams.EnableUserTopicAggregatesFeatureHydration, + PushParams.EnableHtlUserAuthorRTAFeaturesFromFeatureStoreHydration, + PushParams.EnableDurationSinceLastVisitFeatures, + PushParams.EnableTweetAnnotationFeaturesHydration, + PushParams.EnableSpaceVisibilityLibraryFiltering, + PushParams.EnableUserTopicFollowFeatureSetHydration, + PushParams.EnableOnboardingNewUserFeatureSetHydration, + PushParams.EnableMrUserAuthorSparseContFeatureSetHydration, + PushParams.EnableMrUserTopicSparseContFeatureSetHydration, + PushParams.EnableUserPenguinLanguageFeatureSetHydration, + PushParams.EnableMrUserHashspaceEmbeddingFeatureHydration, + PushParams.EnableMrUserEngagedTweetTokensFeatureHydration, + PushParams.EnableMrCandidateTweetTokensFeatureHydration, + PushParams.EnableMrTweetSentimentFeatureHydration, + PushParams.EnableMrTweetAuthorAggregatesFeatureHydration, + PushParams.EnableUserGeoFeatureSetHydration, + PushParams.EnableAuthorGeoFeatureSetHydration, + PushParams.EnableTwHINUserEngagementFeaturesHydration, + PushParams.EnableTwHINUserFollowFeaturesHydration, + PushParams.EnableTwHINAuthorFollowFeaturesHydration, + PushParams.EnableAuthorFollowTwhinEmbeddingFeatureHydration, + PushParams.RampupUserGeoFeatureSetHydration, + PushParams.RampupAuthorGeoFeatureSetHydration, + PushParams.EnablePredicateDetailedInfoScribing, + PushParams.EnablePushCapInfoScribing, + PushParams.EnableUserSignalLanguageFeatureHydration, + PushParams.EnableUserPreferredLanguageFeatureHydration, + PushParams.PopGeoCandidatesDecider, + PushParams.TrendsCandidateDecider, + PushParams.EnableInsTrafficDecider, + PushParams.EnableModelBasedPushcapAssignments, + PushParams.TripGeoTweetCandidatesDecider, + PushParams.ContentRecommenderMixerAdaptorDecider, + PushParams.GenericCandidateAdaptorDecider, + PushParams.TripGeoTweetContentMixerDarkTrafficDecider, + PushParams.EnableIsTweetTranslatableCheck, + PushParams.EnableMrTweetSimClusterFeatureHydration, + PushParams.EnableTwistlyAggregatesFeatureHydration, + PushParams.EnableTweetTwHINFavFeatureHydration, + PushParams.EnableRealGraphV2FeatureHydration, + PushParams.EnableTweetBeTFeatureHydration, + PushParams.EnableMrOfflineUserTweetTopicAggregateHydration, + PushParams.EnableMrOfflineUserTweetSimClusterAggregateHydration, + PushParams.EnableUserSendTimeFeatureHydration, + PushParams.EnableMrUserUtcSendTimeAggregateFeaturesHydration, + PushParams.EnableMrUserLocalSendTimeAggregateFeaturesHydration, + PushParams.EnableBqmlReportModelPredictionForF1Tweets, + PushParams.EnableUserTwhinEmbeddingFeatureHydration, + PushParams.EnableScribingMLFeaturesAsDataRecord, + PushParams.EnableAuthorVerifiedFeatureHydration, + PushParams.EnableAuthorCreatorSubscriptionFeatureHydration, + PushParams.EnableDirectHydrationForUserFeatures + ) + + private val intFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( + Pushservice.SportsMaxNumberOfPushesInIntervalPerEvent, + Pushservice.SportsMaxNumberOfPushesInInterval, + Pushservice.PushMixerMaxResults, + Pushservice.MaxTrendTweetNotificationsInDuration, + Pushservice.MaxRecommendedTrendsToQuery, + Pushservice.NumberOfMaxEarlybirdInNetworkCandidatesParam, + Pushservice.NumberOfMaxCandidatesToBatchInRFPHTakeStep, + Pushservice.MaxMrPushSends24HoursParam, + Pushservice.MaxMrPushSends24HoursNtabOnlyUsersParam, + Pushservice.NumberOfMaxCrMixerCandidatesParam, + Pushservice.RestrictStepSize, + Pushservice.MagicFanoutRankErgThresholdHeavy, + Pushservice.MagicFanoutRankErgThresholdNonHeavy, + Pushservice.MagicFanoutRelaxedEventIdFatigueIntervalInHours, + Pushservice.NumberOfMaxUTEGCandidatesQueriedParam, + Pushservice.HTLVisitFatigueTime, + Pushservice.MaxOnboardingPushInInterval, + Pushservice.MaxTopTweetsByGeoPushGivenInterval, + Pushservice.MaxHighQualityTweetsPushGivenInterval, + Pushservice.MaxTopTweetsByGeoCandidatesToTake, + Pushservice.SpaceRecsRealgraphThreshold, + Pushservice.SpaceRecsGlobalPushLimit, + Pushservice.OptoutExptPushCapParam, + Pushservice.MaxTopTweetImpressionsNotifications, + Pushservice.TopTweetImpressionsMinRequired, + Pushservice.TopTweetImpressionsThreshold, + Pushservice.TopTweetImpressionsOriginalTweetsNumDaysSearch, + Pushservice.TopTweetImpressionsMinNumOriginalTweets, + Pushservice.TopTweetImpressionsMaxFavoritesPerTweet, + Pushservice.TopTweetImpressionsTotalInboundFavoritesLimit, + Pushservice.TopTweetImpressionsTotalFavoritesLimitNumDaysSearch, + Pushservice.TopTweetImpressionsRecentTweetsByAuthorStoreMaxResults, + Pushservice.ANNEfQuery, + Pushservice.NumberOfMaxMrModelingBasedCandidates, + Pushservice.ThresholdOfFavMrModelingBasedCandidates, + Pushservice.LightRankingNumberOfCandidatesParam, + Pushservice.NumberOfDeTopicTweetCandidates, + Pushservice.NumberOfMaxDeTopicTweetCandidatesReturned, + Pushservice.OverrideNotificationsMaxNumOfSlots, + Pushservice.OverrideNotificationsMaxCountForNTab, + Pushservice.MFMaxNumberOfPushesInInterval, + Pushservice.SpacesTopKSimClusterCount, + Pushservice.SpaceRecsSimClusterUserMinimumFollowerCount, + Pushservice.OONSpaceRecsPushLimit, + Pushservice.MagicFanoutRealgraphRankThreshold, + Pushservice.CustomizedPushCapOffset, + Pushservice.NumberOfF1CandidatesThresholdForOONBackfill, + Pushservice.MinimumAllowedAuthorAccountAgeInHours, + Pushservice.RestrictedMinModelPushcap, + Pushservice.ListRecommendationsGeoHashLength, + Pushservice.ListRecommendationsSubscriberCount, + Pushservice.MaxListRecommendationsPushGivenInterval, + Pushservice.SendTimeByUserHistoryMaxOpenedThreshold, + Pushservice.SendTimeByUserHistoryNoSendsHours, + Pushservice.SendTimeByUserHistoryQuickSendBeforeHours, + Pushservice.SendTimeByUserHistoryQuickSendAfterHours, + Pushservice.SendTimeByUserHistoryQuickSendMinDurationInMinute, + Pushservice.SendTimeByUserHistoryNoSendMinDuration, + Pushservice.F1EmojiCopyNumOfPushesFatigue, + Pushservice.OonEmojiCopyNumOfPushesFatigue, + Pushservice.TripTweetMaxTotalCandidates, + Pushservice.InlineFeedbackSubstitutePosition, + Pushservice.HighQualityCandidatesNumberOfCandidates, + Pushservice.HighQualityCandidatesMinNumOfCandidatesToFallback, + Pushservice.ProductLaunchMaxNumberOfPushesInInterval, + Pushservice.CreatorSubscriptionPushMaxNumberOfPushesInInterval, + Pushservice.NewCreatorPushMaxNumberOfPushesInInterval, + Pushservice.TweetReplytoLikeRatioReplyCountThreshold, + Pushservice.MaxExploreVideoTweets, + ) + + private val doubleFeatureSwitchOverrides = + FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( + Pushservice.PercentileThresholdCohort1, + Pushservice.PercentileThresholdCohort2, + Pushservice.PercentileThresholdCohort3, + Pushservice.PercentileThresholdCohort4, + Pushservice.PercentileThresholdCohort5, + Pushservice.PercentileThresholdCohort6, + Pushservice.PnsfwTweetTextThreshold, + Pushservice.PnsfwTweetTextBucketingThreshold, + Pushservice.PnsfwTweetMediaThreshold, + Pushservice.PnsfwTweetImageThreshold, + Pushservice.PnsfwQuoteTweetThreshold, + Pushservice.PnsfwTweetMediaBucketingThreshold, + Pushservice.AgathaCalibratedNSFWThreshold, + Pushservice.AgathaCalibratedNSFWThresholdForMrTwistly, + Pushservice.AgathaTextNSFWThreshold, + Pushservice.AgathaTextNSFWThresholdForMrTwistly, + Pushservice.AgathaCalibratedNSFWBucketThreshold, + Pushservice.AgathaTextNSFWBucketThreshold, + Pushservice.BucketOptoutThresholdParam, + Pushservice.TweetMediaSensitiveCategoryThresholdParam, + Pushservice.CandidateGenerationModelCosineThreshold, + Pushservice.MrModelingBasedCandidatesTopicScoreThreshold, + Pushservice.HashspaceCandidatesTopicScoreThreshold, + Pushservice.FrsTweetCandidatesTopicScoreThreshold, + Pushservice.TopicProofTweetCandidatesTopicScoreThreshold, + Pushservice.SpacesTargetingSimClusterDotProductThreshold, + Pushservice.SautOonWithMediaTweetLengthThresholdParam, + Pushservice.NonSautOonWithMediaTweetLengthThresholdParam, + Pushservice.SautOonWithoutMediaTweetLengthThresholdParam, + Pushservice.NonSautOonWithoutMediaTweetLengthThresholdParam, + Pushservice.ArgfOonWithMediaTweetWordLengthThresholdParam, + Pushservice.EsfthOonWithMediaTweetWordLengthThresholdParam, + Pushservice.BqmlQualityModelPredicateThresholdParam, + Pushservice.LightRankingScribeCandidatesDownSamplingParam, + Pushservice.QualityUprankingBoostForHeavyRankingParam, + Pushservice.QualityUprankingSigmoidBiasForHeavyRankingParam, + Pushservice.QualityUprankingSigmoidWeightForHeavyRankingParam, + Pushservice.QualityUprankingLinearBarForHeavyRankingParam, + Pushservice.QualityUprankingBoostForHighQualityProducersParam, + Pushservice.QualityUprankingDownboostForLowQualityProducersParam, + Pushservice.BqmlHealthModelPredicateFilterThresholdParam, + Pushservice.BqmlHealthModelPredicateBucketThresholdParam, + Pushservice.PNegMultimodalPredicateModelThresholdParam, + Pushservice.PNegMultimodalPredicateBucketThresholdParam, + Pushservice.SeeLessOftenF1TriggerF1PushCapWeight, + Pushservice.SeeLessOftenF1TriggerNonF1PushCapWeight, + Pushservice.SeeLessOftenNonF1TriggerF1PushCapWeight, + Pushservice.SeeLessOftenNonF1TriggerNonF1PushCapWeight, + Pushservice.SeeLessOftenTripHqTweetTriggerF1PushCapWeight, + Pushservice.SeeLessOftenTripHqTweetTriggerNonF1PushCapWeight, + Pushservice.SeeLessOftenTripHqTweetTriggerTripHqTweetPushCapWeight, + Pushservice.SeeLessOftenNtabOnlyNotifUserPushCapWeight, + Pushservice.PromptFeedbackF1TriggerF1PushCapWeight, + Pushservice.PromptFeedbackF1TriggerNonF1PushCapWeight, + Pushservice.PromptFeedbackNonF1TriggerF1PushCapWeight, + Pushservice.PromptFeedbackNonF1TriggerNonF1PushCapWeight, + Pushservice.InlineFeedbackF1TriggerF1PushCapWeight, + Pushservice.InlineFeedbackF1TriggerNonF1PushCapWeight, + Pushservice.InlineFeedbackNonF1TriggerF1PushCapWeight, + Pushservice.InlineFeedbackNonF1TriggerNonF1PushCapWeight, + Pushservice.TweetNtabDislikeCountThresholdParam, + Pushservice.TweetNtabDislikeRateThresholdParam, + Pushservice.TweetNtabDislikeCountThresholdForMrTwistlyParam, + Pushservice.TweetNtabDislikeRateThresholdForMrTwistlyParam, + Pushservice.TweetNtabDislikeCountBucketThresholdParam, + Pushservice.MinAuthorSendsThresholdParam, + Pushservice.MinTweetSendsThresholdParam, + Pushservice.AuthorDislikeRateThresholdParam, + Pushservice.AuthorReportRateThresholdParam, + Pushservice.FavOverSendThresholdParam, + Pushservice.SpreadControlRatioParam, + Pushservice.TweetQTtoNtabClickRatioThresholdParam, + Pushservice.TweetReplytoLikeRatioThresholdLowerBound, + Pushservice.TweetReplytoLikeRatioThresholdUpperBound, + Pushservice.AuthorSensitiveMediaFilteringThreshold, + Pushservice.AuthorSensitiveMediaFilteringThresholdForMrTwistly, + Pushservice.MrRequestScribingEpsGreedyExplorationRatio, + Pushservice.SeeLessOftenTopicTriggerTopicPushCapWeight, + Pushservice.SeeLessOftenTopicTriggerF1PushCapWeight, + Pushservice.SeeLessOftenTopicTriggerOONPushCapWeight, + Pushservice.SeeLessOftenF1TriggerTopicPushCapWeight, + Pushservice.SeeLessOftenOONTriggerTopicPushCapWeight, + Pushservice.SeeLessOftenDefaultPushCapWeight, + Pushservice.OverrideMaxSlotFnWeight, + Pushservice.QualityPredicateExplicitThresholdParam, + Pushservice.AuthorSensitiveScoreWeightInReranking, + Pushservice.BigFilteringThresholdParam, + Pushservice.NsfwScoreThresholdForF1Copy, + Pushservice.NsfwScoreThresholdForOONCopy, + Pushservice.HighOONCThresholdForCopy, + Pushservice.LowOONCThresholdForCopy, + Pushservice.UserDeviceLanguageThresholdParam, + Pushservice.UserInferredLanguageThresholdParam, + Pushservice.SpammyTweetOonThreshold, + Pushservice.SpammyTweetInThreshold, + Pushservice.SpammyTweetBucketingThreshold, + Pushservice.NumFollowerThresholdForHealthAndQualityFilters, + Pushservice.NumFollowerThresholdForHealthAndQualityFiltersPreranking, + Pushservice.SoftRankFactorForSubscriptionCreators, + Pushservice.MagicFanoutSimClusterDotProductHeavyUserThreshold, + Pushservice.MagicFanoutSimClusterDotProductNonHeavyUserThreshold + ) + + private val doubleSeqFeatureSwitchOverrides = + FeatureSwitchOverrideUtil.getDoubleSeqFSOverrides( + Pushservice.MfGridSearchThresholdsCohort1, + Pushservice.MfGridSearchThresholdsCohort2, + Pushservice.MfGridSearchThresholdsCohort3, + Pushservice.MfGridSearchThresholdsCohort4, + Pushservice.MfGridSearchThresholdsCohort5, + Pushservice.MfGridSearchThresholdsCohort6, + Pushservice.MrPercentileGridSearchThresholdsCohort1, + Pushservice.MrPercentileGridSearchThresholdsCohort2, + Pushservice.MrPercentileGridSearchThresholdsCohort3, + Pushservice.MrPercentileGridSearchThresholdsCohort4, + Pushservice.MrPercentileGridSearchThresholdsCohort5, + Pushservice.MrPercentileGridSearchThresholdsCohort6, + Pushservice.GlobalOptoutThresholdParam, + Pushservice.BucketOptoutSlotThresholdParam, + Pushservice.BqmlQualityModelBucketThresholdListParam, + Pushservice.SeeLessOftenListOfDayKnobs, + Pushservice.SeeLessOftenListOfPushCapWeightKnobs, + Pushservice.SeeLessOftenListOfPowerKnobs, + Pushservice.PromptFeedbackListOfDayKnobs, + Pushservice.PromptFeedbackListOfPushCapWeightKnobs, + Pushservice.PromptFeedbackListOfPowerKnobs, + Pushservice.InlineFeedbackListOfDayKnobs, + Pushservice.InlineFeedbackListOfPushCapWeightKnobs, + Pushservice.InlineFeedbackListOfPowerKnobs, + Pushservice.OverrideMaxSlotFnPushCapKnobs, + Pushservice.OverrideMaxSlotFnPowerKnobs, + Pushservice.OverrideMaxSlotFnPushCapKnobs, + Pushservice.MagicRecsRelevanceScoreRange, + Pushservice.MagicFanoutRelevanceScoreRange, + Pushservice.MultilingualPnsfwTweetTextBucketingThreshold, + Pushservice.MultilingualPnsfwTweetTextFilteringThreshold, + ) + + private val booleanFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( + Pushservice.EnablePushRecommendationsParam, + Pushservice.DisableHeavyRankingModelFSParam, + Pushservice.EnablePushMixerReplacingAllSources, + Pushservice.EnablePushMixerReplacingAllSourcesWithControl, + Pushservice.EnablePushMixerReplacingAllSourcesWithExtra, + Pushservice.EnablePushMixerSource, + Common.EnableScheduledSpaceSpeakers, + Common.EnableScheduledSpaceSubscribers, + Pushservice.MagicFanoutNewsUserGeneratedEventsEnable, + Pushservice.MagicFanoutSkipAccountCountryPredicate, + Pushservice.MagicFanoutNewsEnableDescriptionCopy, + Pushservice.EnableF1TriggerSeeLessOftenFatigue, + Pushservice.EnableNonF1TriggerSeeLessOftenFatigue, + Pushservice.AdjustTripHqTweetTriggeredNtabCaretClickFatigue, + Pushservice.EnableCuratedTrendTweets, + Pushservice.EnableNonCuratedTrendTweets, + Pushservice.DisableMlInFilteringFeatureSwitchParam, + Pushservice.EnableTopicCopyForMF, + Pushservice.EnableTopicCopyForImplicitTopics, + Pushservice.EnableRestrictStep, + Pushservice.EnableHighPriorityPush, + Pushservice.BoostCandidatesFromSubscriptionCreators, + Pushservice.SoftRankCandidatesFromSubscriptionCreators, + Pushservice.EnableNewMROONCopyForPush, + Pushservice.EnableQueryAuthorMediaRepresentationStore, + Pushservice.EnableProfanityFilterParam, + Pushservice.EnableAbuseStrikeTop2PercentFilterSimCluster, + Pushservice.EnableAbuseStrikeTop1PercentFilterSimCluster, + Pushservice.EnableAbuseStrikeTop05PercentFilterSimCluster, + Pushservice.EnableAgathaUserHealthModelPredicate, + Pushservice.PnsfwTweetMediaFilterOonOnly, + Pushservice.EnableHealthSignalStorePnsfwTweetTextPredicate, + Pushservice.EnableHealthSignalStoreMultilingualPnsfwTweetTextPredicate, + Pushservice.DisableHealthFiltersForCrMixerCandidates, + Pushservice.EnableOverrideNotificationsForAndroid, + Pushservice.EnableOverrideNotificationsForIos, + Pushservice.EnableMrRequestScribingForTargetFiltering, + Pushservice.EnableMrRequestScribingForCandidateFiltering, + Pushservice.EnableMrRequestScribingWithFeatureHydrating, + Pushservice.EnableFlattenMrRequestScribing, + Pushservice.EnableMrRequestScribingForEpsGreedyExploration, + Pushservice.EnableMrRequestScribingDismissScore, + Pushservice.EnableMrRequestScribingBigFilteringSupervisedScores, + Pushservice.EnableMrRequestScribingBigFilteringRLScores, + Pushservice.EnableEventPrimaryMediaAndroid, + Pushservice.EnableEventSquareMediaIosMagicFanoutNewsEvent, + Pushservice.EnableEventSquareMediaAndroid, + Pushservice.EnableMagicFanoutNewsForYouNtabCopy, + Pushservice.EnableMfGeoTargeting, + Pushservice.EnableRuxLandingPage, + Pushservice.EnableNTabRuxLandingPage, + Pushservice.EnableGraduallyRampUpNotification, + Pushservice.EnableOnboardingPushes, + Pushservice.EnableAddressBookPush, + Pushservice.EnableCompleteOnboardingPush, + Pushservice.EnableOverrideNotificationsSmartPushConfigForAndroid, + Pushservice.DisableOnboardingPushFatigue, + Pushservice.EnableTopTweetsByGeoCandidates, + Pushservice.BackfillRankTopTweetsByGeoCandidates, + Pushservice.PopGeoTweetEnableAggressiveThresholds, + Pushservice.EnableMrMinDurationSinceMrPushFatigue, + Pushservice.EnableF1FromProtectedTweetAuthors, + Pushservice.MagicFanoutEnableCustomTargetingNewsEvent, + Pushservice.EnableSafeUserTweetTweetypieStore, + Pushservice.EnableMrMinDurationSinceMrPushFatigue, + Pushservice.EnableHydratingOnlineMRHistoryFeatures, + Common.SpaceRecsEnableHostNotifs, + Common.SpaceRecsEnableSpeakerNotifs, + Common.SpaceRecsEnableListenerNotifs, + Common.EnableMagicFanoutProductLaunch, + Pushservice.EnableTopTweetsByGeoCandidatesForDormantUsers, + Pushservice.EnableOverrideNotificationsScoreBasedOverride, + Pushservice.EnableOverrideNotificationsMultipleTargetIds, + Pushservice.EnableMinDurationModifier, + Pushservice.EnableMinDurationModifierV2, + Pushservice.EnableMinDurationModifierByUserHistory, + Pushservice.EnableQueryUserOpenedHistory, + Pushservice.EnableRandomHourForQuickSend, + Pushservice.EnableFrsCandidates, + Pushservice.EnableFrsTweetCandidatesTopicSetting, + Pushservice.EnableFrsTweetCandidatesTopicAnnotation, + Pushservice.EnableFrsTweetCandidatesTopicCopy, + Pushservice.EnableCandidateGenerationModelParam, + Pushservice.EnableOverrideForSportsCandidates, + Pushservice.EnableEventIdBasedOverrideForSportsCandidates, + Pushservice.EnableMrModelingBasedCandidates, + Pushservice.EnableMrModelingBasedCandidatesTopicSetting, + Pushservice.EnableMrModelingBasedCandidatesTopicAnnotation, + Pushservice.EnableMrModelingBasedCandidatesTopicCopy, + Pushservice.EnableResultFromFrsCandidates, + Pushservice.EnableHashspaceCandidates, + Pushservice.EnableHashspaceCandidatesTopicSetting, + Pushservice.EnableHashspaceCandidatesTopicAnnotation, + Pushservice.EnableHashspaceCandidatesTopicCopy, + Pushservice.EnableResultFromHashspaceCandidates, + Pushservice.EnableDownRankOfNewUserPlaybookTopicFollowPush, + Pushservice.EnableDownRankOfNewUserPlaybookTopicTweetPush, + Pushservice.EnableTopTweetImpressionsNotification, + Pushservice.EnableLightRankingParam, + Pushservice.EnableRandomBaselineLightRankingParam, + Pushservice.EnableQualityUprankingForHeavyRankingParam, + Pushservice.EnableQualityUprankingCrtScoreStatsForHeavyRankingParam, + Pushservice.EnableProducersQualityBoostingForHeavyRankingParam, + Pushservice.EnableMrScribingMLFeaturesAsFeatureMapForStaging, + Pushservice.EnableMrTweetSentimentFeatureHydrationFS, + Pushservice.EnableTimelineHealthSignalHydration, + Pushservice.EnableTopicEngagementRealTimeAggregatesFS, + Pushservice.EnableMrUserSemanticCoreFeatureForExpt, + Pushservice.EnableHydratingRealGraphTargetUserFeatures, + Pushservice.EnableHydratingUserDurationSinceLastVisitFeatures, + Pushservice.EnableRealGraphUserAuthorAndSocialContxtFeatureHydration, + Pushservice.EnableUserTopicAggregatesFS, + Pushservice.EnableTimelineHealthSignalHydrationForModelTraining, + Pushservice.EnableMrUserSocialContextAggregateFeatureHydration, + Pushservice.EnableMrUserSemanticCoreAggregateFeatureHydration, + Pushservice.EnableMrUserCandidateSparseOfflineAggregateFeatureHydration, + Pushservice.EnableMrUserCandidateOfflineAggregateFeatureHydration, + Pushservice.EnableMrUserCandidateOfflineCompactAggregateFeatureHydration, + Pushservice.EnableMrUserAuthorOfflineAggregateFeatureHydration, + Pushservice.EnableMrUserAuthorOfflineCompactAggregateFeatureHydration, + Pushservice.EnableMrUserOfflineCompactAggregateFeatureHydration, + Pushservice.EnableMrUserSimcluster2020AggregateFeatureHydration, + Pushservice.EnableMrUserOfflineAggregateFeatureHydration, + Pushservice.EnableBqmlQualityModelPredicateParam, + Pushservice.EnableBqmlQualityModelScoreHistogramParam, + Pushservice.EnableBqmlHealthModelPredicateParam, + Pushservice.EnableBqmlHealthModelPredictionForInNetworkCandidatesParam, + Pushservice.EnableBqmlHealthModelScoreHistogramParam, + Pushservice.EnablePNegMultimodalPredicateParam, + Pushservice.EnableNegativeKeywordsPredicateParam, + Pushservice.EnableTweetAuthorAggregatesFeatureHydrationParam, + Pushservice.OonTweetLengthPredicateUpdatedMediaLogic, + Pushservice.OonTweetLengthPredicateUpdatedQuoteTweetLogic, + Pushservice.OonTweetLengthPredicateMoreStrictForUndefinedLanguages, + Pushservice.EnablePrerankingTweetLengthPredicate, + Pushservice.EnableDeTopicTweetCandidates, + Pushservice.EnableDeTopicTweetCandidateResults, + Pushservice.EnableDeTopicTweetCandidatesCustomTopics, + Pushservice.EnableDeTopicTweetCandidatesCustomLanguages, + Pushservice.EnableMrTweetSimClusterFeatureHydrationFS, + Pushservice.DisableOutNetworkTweetCandidatesFS, + Pushservice.EnableLaunchVideosInImmersiveExplore, + Pushservice.EnableStoringNtabGenericNotifKey, + Pushservice.EnableDeletingNtabTimeline, + Pushservice.EnableOverrideNotificationsNSlots, + Pushservice.EnableNslotsForOverrideOnNtab, + Pushservice.EnableOverrideMaxSlotFn, + Pushservice.EnableTargetIdInSmartPushPayloadForMagicFanoutSportsEvent, + Pushservice.EnableOverrideIdNTabRequest, + Pushservice.EnableOverrideForSpaces, + Pushservice.EnableTopicProofTweetRecs, + Pushservice.EnableHealthFiltersForTopicProofTweet, + Pushservice.EnableTargetIdsInSmartPushPayload, + Pushservice.EnableSecondaryAccountPredicateMF, + Pushservice.EnableInlineVideo, + Pushservice.EnableAutoplayForInlineVideo, + Pushservice.EnableOONGeneratedInlineActions, + Pushservice.EnableInlineFeedbackOnPush, + Pushservice.UseInlineActionsV1, + Pushservice.UseInlineActionsV2, + Pushservice.EnableFeaturedSpacesOON, + Pushservice.CheckFeaturedSpaceOON, + Pushservice.EnableGeoTargetingForSpaces, + Pushservice.EnableEmployeeOnlySpaceNotifications, + Pushservice.EnableSpacesTtlForNtab, + Pushservice.EnableCustomThreadIdForOverride, + Pushservice.EnableSimClusterTargetingSpaces, + Pushservice.TargetInInlineActionAppVisitFatigue, + Pushservice.EnableInlineActionAppVisitFatigue, + Pushservice.EnableThresholdOfFavMrModelingBasedCandidates, + Pushservice.HydrateMrUserSimclusterV2020InModelingBasedCG, + Pushservice.HydrateMrUserSemanticCoreInModelingBasedCG, + Pushservice.HydrateOnboardingInModelingBasedCG, + Pushservice.HydrateTopicFollowInModelingBasedCG, + Pushservice.HydrateMrUserTopicInModelingBasedCG, + Pushservice.HydrateMrUserAuthorInModelingBasedCG, + Pushservice.HydrateUserPenguinLanguageInModelingBasedCG, + Pushservice.EnableMrUserEngagedTweetTokensFeature, + Pushservice.HydrateMrUserHashspaceEmbeddingInModelingBasedCG, + Pushservice.HydrateUseGeoInModelingBasedCG, + Pushservice.EnableSpaceCohostJoinEvent, + Pushservice.EnableOONFilteringBasedOnUserSettings, + Pushservice.EnableContFnF1TriggerSeeLessOftenFatigue, + Pushservice.EnableContFnNonF1TriggerSeeLessOftenFatigue, + Pushservice.EnableContFnF1TriggerPromptFeedbackFatigue, + Pushservice.EnableContFnNonF1TriggerPromptFeedbackFatigue, + Pushservice.EnableContFnF1TriggerInlineFeedbackFatigue, + Pushservice.EnableContFnNonF1TriggerInlineFeedbackFatigue, + Pushservice.UseInlineDislikeForFatigue, + Pushservice.UseInlineDismissForFatigue, + Pushservice.UseInlineSeeLessForFatigue, + Pushservice.UseInlineNotRelevantForFatigue, + Pushservice.GPEnableCustomMagicFanoutCricketFatigue, + Pushservice.IncludeRelevanceScoreInIbis2Payload, + Pushservice.BypassGlobalSpacePushCapForSoftDeviceFollow, + Pushservice.EnableCountryCodeBackoffTopTweetsByGeo, + Pushservice.EnableNewCreatorPush, + Pushservice.EnableCreatorSubscriptionPush, + Pushservice.EnableInsSender, + Pushservice.EnableOptoutAdjustedPushcap, + Pushservice.EnableOONBackfillBasedOnF1Candidates, + Pushservice.EnableVFInTweetypie, + Pushservice.EnablePushPresentationVerifiedSymbol, + Pushservice.EnableHighPrioritySportsPush, + Pushservice.EnableSearchURLRedirectForSportsFanout, + Pushservice.EnableScoreFanoutNotification, + Pushservice.EnableExplicitPushCap, + Pushservice.EnableNsfwTokenBasedFiltering, + Pushservice.EnableRestrictedMinModelPushcap, + Pushservice.EnableGenericCRTBasedFatiguePredicate, + Pushservice.EnableCopyFeaturesForF1, + Pushservice.EnableEmojiInF1Copy, + Pushservice.EnableTargetInF1Copy, + Pushservice.EnableCopyFeaturesForOon, + Pushservice.EnableEmojiInOonCopy, + Pushservice.EnableTargetInOonCopy, + Pushservice.EnableF1CopyBody, + Pushservice.EnableOONCopyBody, + Pushservice.EnableIosCopyBodyTruncate, + Pushservice.EnableHTLBasedFatigueBasicRule, + Pushservice.EnableTargetAndEmojiSplitFatigue, + Pushservice.EnableNsfwCopy, + Pushservice.EnableOONCopyBody, + Pushservice.EnableTweetDynamicInlineActions, + Pushservice.EnablePushcapRefactor, + Pushservice.BigFilteringEnableHistogramsParam, + Pushservice.EnableTweetTranslation, + Pushservice.TripTweetCandidateReturnEnable, + Pushservice.EnableSocialContextForRetweet, + Pushservice.EnableEmptyBody, + Pushservice.EnableLocalViralTweets, + Pushservice.EnableExploreVideoTweets, + Pushservice.EnableDynamicInlineActionsForDesktopWeb, + Pushservice.EnableDynamicInlineActionsForMobileWeb, + Pushservice.EnableNTabEntriesForSportsEventNotifications, + Pushservice.EnableNTabFacePileForSportsEventNotifications, + Pushservice.DisableIsTargetBlueVerifiedPredicate, + Pushservice.EnableNTabEntriesForProductLaunchNotifications, + Pushservice.DisableIsTargetLegacyVerifiedPredicate, + Pushservice.EnableNTabOverrideForSportsEventNotifications, + Pushservice.EnableOONCBasedCopy, + Pushservice.HighQualityCandidatesEnableCandidateSource, + Pushservice.HighQualityCandidatesEnableFallback, + Pushservice.EnableTweetLanguageFilter, + Pushservice.EnableListRecommendations, + Pushservice.EnableIDSListRecommendations, + Pushservice.EnablePopGeoListRecommendations, + Pushservice.SkipLanguageFilterForMediaTweets, + Pushservice.EnableSpammyTweetFilter, + Pushservice.EnableTweetPushToHomeAndroid, + Pushservice.EnableTweetPushToHomeiOS, + Pushservice.EnableBoundedFeatureSetForSocialContext, + Pushservice.EnableStpBoundedFeatureSetForUserSocialContext, + Pushservice.EnableCoreUserHistoryBoundedFeatureSetForSocialContext, + Pushservice.SkipPostRankingFilters, + Pushservice.MRWebHoldbackParam, + Pushservice.EnableIsTargetSuperFollowCreatorPredicate + ) + + private val longSeqFeatureSwitchOverrides = + FeatureSwitchOverrideUtil.getLongSeqFSOverrides( + Pushservice.MagicFanoutEventAllowlistToSkipAccountCountryPredicate + ) + + private val longSetFeatureSwitchOverrides = + FeatureSwitchOverrideUtil.getLongSetFSOverrides( + Pushservice.ListOfAdhocIdsForStatsTracking + ) + + private val stringSeqFeatureSwitchOverrides = + FeatureSwitchOverrideUtil.getStringSeqFSOverrides( + Pushservice.ListOfCrtsForOpenApp, + Pushservice.ListOfCrtsToUpRank, + Pushservice.OONCandidatesDisabledCrTagParam, + Pushservice.ListOfCrtsToDownRank, + Pushservice.MagicFanoutDenyListedCountries, + Pushservice.GlobalOptoutModelParam, + Pushservice.BqmlQualityModelBucketModelIdListParam, + Pushservice.CommonRecommendationTypeDenyListPushHoldbacks, + Pushservice.TargetLevelFeatureListForMrRequestScribing, + Pushservice.MagicFanoutSportsEventDenyListedCountries, + Pushservice.MultilingualPnsfwTweetTextSupportedLanguages, + Pushservice.NegativeKeywordsPredicateDenylist, + Pushservice.TripTweetCandidateSourceIds, + Pushservice.NsfwTokensParam, + Pushservice.HighQualityCandidatesFallbackSourceIds + ) + + private val intSeqFeatureSwitchOverrides = + FeatureSwitchOverrideUtil.getIntSeqFSOverrides( + Pushservice.BucketOptoutSlotPushcapParam, + Pushservice.GeoHashLengthList, + Pushservice.MinDurationModifierStartHourList, + Pushservice.MinDurationModifierEndHourList, + Pushservice.MinDurationTimeModifierConst + ) + + private val enumFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( + stat, + logger, + Pushservice.MRBoldTitleFavoriteAndRetweetParam, + Pushservice.QualityUprankingTransformTypeParam, + Pushservice.QualityPredicateIdParam, + Pushservice.BigFilteringNormalizationTypeIdParam, + Common.PushcapModelType, + Common.MFCricketTargetingPredicate, + Pushservice.RankingFunctionForTopTweetsByGeo, + Pushservice.TopTweetsByGeoCombinationParam, + Pushservice.PopGeoTweetVersionParam, + Pushservice.SubtextInAndroidPushHeaderParam, + Pushservice.HighOONCTweetFormat, + Pushservice.LowOONCTweetFormat, + ) + + private val enumSeqFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getEnumSeqFSOverrides( + stat, + logger, + Pushservice.OONTweetDynamicInlineActionsList, + Pushservice.TweetDynamicInlineActionsList, + Pushservice.TweetDynamicInlineActionsListForWeb, + Pushservice.HighQualityCandidatesEnableGroups, + Pushservice.HighQualityCandidatesFallbackEnabledGroups, + Pushservice.OONCandidatesDisabledCrtGroupParam, + Pushservice.MultilingualPnsfwTweetTextBucketingModelList, + ) + + private val stringFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( + Common.PushcapModelPredictionVersion, + Pushservice.WeightedOpenOrNtabClickRankingModelParam, + Pushservice.WeightedOpenOrNtabClickFilteringModelParam, + Pushservice.BucketOptoutModelParam, + Pushservice.ScoringFuncForTopTweetsByGeo, + Pushservice.LightRankingModelTypeParam, + Pushservice.BigFilteringSupervisedSendingModelParam, + Pushservice.BigFilteringSupervisedWithoutSendingModelParam, + Pushservice.BigFilteringRLSendingModelParam, + Pushservice.BigFilteringRLWithoutSendingModelParam, + Pushservice.BqmlQualityModelTypeParam, + Pushservice.BqmlHealthModelTypeParam, + Pushservice.QualityUprankingModelTypeParam, + Pushservice.SearchURLRedirectForSportsFanout, + Pushservice.LocalViralTweetsBucket, + Pushservice.HighQualityCandidatesHeavyRankingModel, + Pushservice.HighQualityCandidatesNonPersonalizedQualityCnnModel, + Pushservice.HighQualityCandidatesBqmlNsfwModel, + Pushservice.HighQualityCandidatesBqmlReportModel, + Pushservice.ProductLaunchLandingPageDeepLink, + Pushservice.ProductLaunchTapThrough, + Pushservice.TweetLanguageFeatureNameParam + ) + + private val durationFeatureSwitchOverrides = + FeatureSwitchOverrideUtil.getBoundedDurationFSOverrides( + Common.NumberOfDaysToFilterMRForSeeLessOften, + Common.NumberOfDaysToReducePushCapForSeeLessOften, + Pushservice.NumberOfDaysToFilterForSeeLessOftenForF1TriggerF1, + Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerF1, + Pushservice.NumberOfDaysToFilterForSeeLessOftenForF1TriggerNonF1, + Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForF1TriggerNonF1, + Pushservice.NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerF1, + Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerF1, + Pushservice.NumberOfDaysToFilterForSeeLessOftenForNonF1TriggerNonF1, + Pushservice.NumberOfDaysToReducePushCapForSeeLessOftenForNonF1TriggerNonF1, + Pushservice.TrendTweetNotificationsFatigueDuration, + Pushservice.MinDurationSincePushParam, + Pushservice.MFMinIntervalFatigue, + Pushservice.SimclusterBasedCandidateMaxTweetAgeParam, + Pushservice.DetopicBasedCandidateMaxTweetAgeParam, + Pushservice.F1CandidateMaxTweetAgeParam, + Pushservice.MaxTweetAgeParam, + Pushservice.ModelingBasedCandidateMaxTweetAgeParam, + Pushservice.GeoPopTweetMaxAgeInHours, + Pushservice.MinDurationSincePushParam, + Pushservice.GraduallyRampUpPhaseDurationDays, + Pushservice.MrMinDurationSincePushForOnboardingPushes, + Pushservice.FatigueForOnboardingPushes, + Pushservice.FrigateHistoryOtherNotificationWriteTtl, + Pushservice.FrigateHistoryTweetNotificationWriteTtl, + Pushservice.TopTweetsByGeoPushInterval, + Pushservice.HighQualityTweetsPushInterval, + Pushservice.MrMinDurationSincePushForTopTweetsByGeoPushes, + Pushservice.TimeSinceLastLoginForGeoPopTweetPush, + Pushservice.NewUserPlaybookAllowedLastLoginHours, + Pushservice.SpaceRecsAppFatigueDuration, + Pushservice.OONSpaceRecsFatigueDuration, + Pushservice.SpaceRecsFatigueMinIntervalDuration, + Pushservice.SpaceRecsGlobalFatigueDuration, + Pushservice.MinimumTimeSinceLastLoginForGeoPopTweetPush, + Pushservice.MinFatigueDurationSinceLastHTLVisit, + Pushservice.LastHTLVisitBasedNonFatigueWindow, + Pushservice.SpaceNotificationsTTLDurationForNTab, + Pushservice.OverrideNotificationsLookbackDurationForOverrideInfo, + Pushservice.OverrideNotificationsLookbackDurationForImpressionId, + Pushservice.OverrideNotificationsLookbackDurationForNTab, + Pushservice.TopTweetImpressionsNotificationInterval, + Pushservice.TopTweetImpressionsFatigueMinIntervalDuration, + Pushservice.MFPushIntervalInHours, + Pushservice.InlineActionAppVisitFatigue, + Pushservice.SpaceParticipantHistoryLastActiveThreshold, + Pushservice.SportsMinIntervalFatigue, + Pushservice.SportsPushIntervalInHours, + Pushservice.SportsMinIntervalFatiguePerEvent, + Pushservice.SportsPushIntervalInHoursPerEvent, + Pushservice.TargetNtabOnlyCapFatigueIntervalHours, + Pushservice.TargetPushCapFatigueIntervalHours, + Pushservice.CopyFeaturesHistoryLookbackDuration, + Pushservice.F1EmojiCopyFatigueDuration, + Pushservice.F1TargetCopyFatigueDuration, + Pushservice.OonEmojiCopyFatigueDuration, + Pushservice.OonTargetCopyFatigueDuration, + Pushservice.ProductLaunchPushIntervalInHours, + Pushservice.ExploreVideoTweetAgeParam, + Pushservice.ListRecommendationsPushInterval, + Pushservice.ProductLaunchMinIntervalFatigue, + Pushservice.NewCreatorPushIntervalInHours, + Pushservice.NewCreatorPushMinIntervalFatigue, + Pushservice.CreatorSubscriptionPushIntervalInHours, + Pushservice.CreatorSubscriptionPushhMinIntervalFatigue + ) + + private[params] val allFeatureSwitchOverrides = + booleanDeciderOverrides ++ + booleanFeatureSwitchOverrides ++ + intFeatureSwitchOverrides ++ + doubleFeatureSwitchOverrides ++ + doubleSeqFeatureSwitchOverrides ++ + enumFeatureSwitchOverrides ++ + stringSeqFeatureSwitchOverrides ++ + stringFeatureSwitchOverrides ++ + durationFeatureSwitchOverrides ++ + intSeqFeatureSwitchOverrides ++ + longSeqFeatureSwitchOverrides ++ + enumSeqFeatureSwitchOverrides ++ + longSetFeatureSwitchOverrides + + val config = BaseConfigBuilder(allFeatureSwitchOverrides).build() +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.scala new file mode 100644 index 0000000000..c451a61bcf --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushMLModelParams.scala @@ -0,0 +1,60 @@ +package com.twitter.frigate.pushservice.params + +/** + * This enum defines ML models for push + */ +object PushMLModel extends Enumeration { + type PushMLModel = Value + + val WeightedOpenOrNtabClickProbability = Value + val DauProbability = Value + val OptoutProbability = Value + val FilteringProbability = Value + val BigFilteringSupervisedSendingModel = Value + val BigFilteringSupervisedWithoutSendingModel = Value + val BigFilteringRLSendingModel = Value + val BigFilteringRLWithoutSendingModel = Value + val HealthNsfwProbability = Value +} + +object WeightedOpenOrNtabClickModel { + type ModelNameType = String + + // MR models + val Periodically_Refreshed_Prod_Model = + "Periodically_Refreshed_Prod_Model" // used in DBv2 service, needed for gradually migrate via feature switch +} + + +object OptoutModel { + type ModelNameType = String + val D0_has_realtime_features = "D0_has_realtime_features" + val D0_no_realtime_features = "D0_no_realtime_features" +} + +object HealthNsfwModel { + type ModelNameType = String + val Q2_2022_Mr_Bqml_Health_Model_NsfwV0 = "Q2_2022_Mr_Bqml_Health_Model_NsfwV0" +} + +object BigFilteringSupervisedModel { + type ModelNameType = String + val V0_0_BigFiltering_Supervised_Sending_Model = "Q3_2022_bigfiltering_supervised_send_model_v0" + val V0_0_BigFiltering_Supervised_Without_Sending_Model = + "Q3_2022_bigfiltering_supervised_not_send_model_v0" +} + +object BigFilteringRLModel { + type ModelNameType = String + val V0_0_BigFiltering_Rl_Sending_Model = "Q3_2022_bigfiltering_rl_send_model_dqn_dau_15_open" + val V0_0_BigFiltering_Rl_Without_Sending_Model = + "Q3_2022_bigfiltering_rl_not_send_model_dqn_dau_15_open" +} + +case class PushModelName( + modelType: PushMLModel.Value, + version: WeightedOpenOrNtabClickModel.ModelNameType) { + override def toString: String = { + modelType.toString + "_" + version + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.scala new file mode 100644 index 0000000000..5e5f6af6ab --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushParams.scala @@ -0,0 +1,534 @@ +package com.twitter.frigate.pushservice.params + +import com.twitter.rux.common.context.thriftscala.ExperimentKey +import com.twitter.timelines.configapi.Param +import com.twitter.timelines.configapi.decider.BooleanDeciderParam + +object PushParams { + + /** + * Disable ML models in filtering + */ + object DisableMlInFilteringParam extends BooleanDeciderParam(DeciderKey.disableMLInFiltering) + + /** + * Disable ML models in ranking, use random ranking instead + * This param is used for ML holdback and training data collection + */ + object UseRandomRankingParam extends Param(false) + + /** + * Disable feature hydration, ML ranking, and ML filtering + * Use default order from candidate source + * This param is for service continuity + */ + object DisableAllRelevanceParam extends BooleanDeciderParam(DeciderKey.disableAllRelevance) + + /** + * Disable ML heavy ranking + * Use default order from candidate source + * This param is for service continuity + */ + object DisableHeavyRankingParam extends BooleanDeciderParam(DeciderKey.disableHeavyRanking) + + /** + * Restrict ML light ranking by selecting top3 candidates + * Use default order from candidate source + * This param is for service continuity + */ + object RestrictLightRankingParam extends BooleanDeciderParam(DeciderKey.restrictLightRanking) + + /** + * Downsample ML light ranking scribed candidates + */ + object DownSampleLightRankingScribeCandidatesParam + extends BooleanDeciderParam(DeciderKey.downSampleLightRankingScribeCandidates) + + /** + * Set it to true only for Android only ranking experiments + */ + object AndroidOnlyRankingExperimentParam extends Param(false) + + /** + * Enable the user_tweet_entity_graph tweet candidate source. + */ + object UTEGTweetCandidateSourceParam + extends BooleanDeciderParam(DeciderKey.entityGraphTweetRecsDeciderKey) + + /** + * Enable writes to Notification Service + */ + object EnableWritesToNotificationServiceParam + extends BooleanDeciderParam(DeciderKey.enablePushserviceWritesToNotificationServiceDeciderKey) + + /** + * Enable writes to Notification Service for all employees + */ + object EnableWritesToNotificationServiceForAllEmployeesParam + extends BooleanDeciderParam( + DeciderKey.enablePushserviceWritesToNotificationServiceForAllEmployeesDeciderKey) + + /** + * Enable writes to Notification Service for everyone + */ + object EnableWritesToNotificationServiceForEveryoneParam + extends BooleanDeciderParam( + DeciderKey.enablePushserviceWritesToNotificationServiceForEveryoneDeciderKey) + + /** + * Enable fatiguing MR for Ntab caret click + */ + object EnableFatigueNtabCaretClickingParam extends Param(true) + + /** + * Param for disabling in-network Tweet candidates + */ + object DisableInNetworkTweetCandidatesParam extends Param(false) + + /** + * Decider controlled param to enable prompt feedback response NO predicate + */ + object EnablePromptFeedbackFatigueResponseNoPredicate + extends BooleanDeciderParam( + DeciderKey.enablePromptFeedbackFatigueResponseNoPredicateDeciderKey) + + /** + * Enable hydration and generation of Social context (TF, TR) based candidates for Earlybird Tweets + */ + object EarlyBirdSCBasedCandidatesParam + extends BooleanDeciderParam(DeciderKey.enableUTEGSCForEarlybirdTweetsDecider) + + /** + * Param to allow reduce to one social proof for tweet param in UTEG + */ + object AllowOneSocialProofForTweetInUTEGParam extends Param(true) + + /** + * Param to query UTEG for out network tweets only + */ + object OutNetworkTweetsOnlyForUTEGParam extends Param(false) + + object EnablePushSendEventBus extends BooleanDeciderParam(DeciderKey.enablePushSendEventBus) + + /** + * Enable RUX Tweet landing page for push open on iOS + */ + object EnableRuxLandingPageIOSParam extends Param[Boolean](true) + + /** + * Enable RUX Tweet landing page for push open on Android + */ + object EnableRuxLandingPageAndroidParam extends Param[Boolean](true) + + /** + * Param to decide which ExperimentKey to be encoded into Rux landing page context object. + * The context object is sent to rux-api and rux-api applies logic (e.g. show reply module on + * rux landing page or not) accordingly based on the experiment key. + */ + object RuxLandingPageExperimentKeyIOSParam extends Param[Option[ExperimentKey]](None) + object RuxLandingPageExperimentKeyAndroidParam extends Param[Option[ExperimentKey]](None) + + /** + * Param to enable MR Tweet Fav Recs + */ + object MRTweetFavRecsParam extends BooleanDeciderParam(DeciderKey.enableTweetFavRecs) + + /** + * Param to enable MR Tweet Retweet Recs + */ + object MRTweetRetweetRecsParam extends BooleanDeciderParam(DeciderKey.enableTweetRetweetRecs) + + /** + * Param to disable writing to NTAB + * */ + object DisableWritingToNTAB extends Param[Boolean](default = false) + + /** + * Param to show RUX landing page as a modal on iOS + */ + object ShowRuxLandingPageAsModalOnIOS extends Param[Boolean](default = false) + + /** + * Param to enable mr end to end scribing + */ + object EnableMrRequestScribing extends BooleanDeciderParam(DeciderKey.enableMrRequestScribing) + + /** + * Param to enable scribing of high quality candidate scores + */ + object EnableHighQualityCandidateScoresScribing + extends BooleanDeciderParam(DeciderKey.enableHighQualityCandidateScoresScribing) + + /** + * Decider controlled param to pNeg multimodal predictions for F1 tweets + */ + object EnablePnegMultimodalPredictionForF1Tweets + extends BooleanDeciderParam(DeciderKey.enablePnegMultimodalPredictionForF1Tweets) + + /** + * Decider controlled param to scribe oonFav score for F1 tweets + */ + object EnableScribeOonFavScoreForF1Tweets + extends BooleanDeciderParam(DeciderKey.enableScribingOonFavScoreForF1Tweets) + + /** + * Param to enable htl user aggregates extended hydration + */ + object EnableHtlOfflineUserAggregatesExtendedHydration + extends BooleanDeciderParam(DeciderKey.enableHtlOfflineUserAggregateExtendedFeaturesHydration) + + /** + * Param to enable predicate detailed info scribing + */ + object EnablePredicateDetailedInfoScribing + extends BooleanDeciderParam(DeciderKey.enablePredicateDetailedInfoScribing) + + /** + * Param to enable predicate detailed info scribing + */ + object EnablePushCapInfoScribing + extends BooleanDeciderParam(DeciderKey.enablePredicateDetailedInfoScribing) + + /** + * Param to enable user signal language feature hydration + */ + object EnableUserSignalLanguageFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableUserSignalLanguageFeatureHydration) + + /** + * Param to enable user preferred language feature hydration + */ + object EnableUserPreferredLanguageFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableUserPreferredLanguageFeatureHydration) + + /** + * Param to enable ner erg feature hydration + */ + object EnableNerErgFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableNerErgFeaturesHydration) + + /** + * Param to enable inline action on push copy for Android + */ + object MRAndroidInlineActionOnPushCopyParam extends Param[Boolean](default = true) + + /** + * Param to enable hydrating mr user semantic core embedding features + * */ + object EnableMrUserSemanticCoreFeaturesHydration + extends BooleanDeciderParam(DeciderKey.enableMrUserSemanticCoreFeaturesHydration) + + /** + * Param to enable hydrating mr user semantic core embedding features filtered by 0.0000001 + * */ + object EnableMrUserSemanticCoreNoZeroFeaturesHydration + extends BooleanDeciderParam(DeciderKey.enableMrUserSemanticCoreNoZeroFeaturesHydration) + + /* + * Param to enable days since user's recent resurrection features hydration + */ + object EnableDaysSinceRecentResurrectionFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableDaysSinceRecentResurrectionFeatureHydration) + + /* + * Param to enable days since user past aggregates features hydration + */ + object EnableUserPastAggregatesFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableUserPastAggregatesFeatureHydration) + + /* + * Param to enable mr user simcluster features (v2020) hydration + * */ + object EnableMrUserSimclusterV2020FeaturesHydration + extends BooleanDeciderParam(DeciderKey.enableMrUserSimclusterV2020FeaturesHydration) + + /* + * Param to enable mr user simcluster features (v2020) hydration + * */ + object EnableMrUserSimclusterV2020NoZeroFeaturesHydration + extends BooleanDeciderParam(DeciderKey.enableMrUserSimclusterV2020NoZeroFeaturesHydration) + + /* + * Param to enable HTL topic engagement realtime aggregate features + * */ + object EnableTopicEngagementRealTimeAggregatesFeatureHydration + extends BooleanDeciderParam( + DeciderKey.enableTopicEngagementRealTimeAggregatesFeatureHydration) + + object EnableUserTopicAggregatesFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableUserTopicAggregatesFeatureHydration) + + /** + * Param to enable user author RTA feature hydration + */ + object EnableHtlUserAuthorRTAFeaturesFromFeatureStoreHydration + extends BooleanDeciderParam(DeciderKey.enableHtlUserAuthorRealTimeAggregateFeatureHydration) + + /** + * Param to enable duration since last visit features + */ + object EnableDurationSinceLastVisitFeatures + extends BooleanDeciderParam(DeciderKey.enableDurationSinceLastVisitFeatureHydration) + + object EnableTweetAnnotationFeaturesHydration + extends BooleanDeciderParam(DeciderKey.enableTweetAnnotationFeatureHydration) + + /** + * Param to Enable visibility filtering through SpaceVisibilityLibrary from SpacePredicate + */ + object EnableSpaceVisibilityLibraryFiltering + extends BooleanDeciderParam(DeciderKey.enableSpaceVisibilityLibraryFiltering) + + /* + * Param to enable user topic follow feature set hydration + * */ + object EnableUserTopicFollowFeatureSetHydration + extends BooleanDeciderParam(DeciderKey.enableUserTopicFollowFeatureSet) + + /* + * Param to enable onboarding new user feature set hydration + * */ + object EnableOnboardingNewUserFeatureSetHydration + extends BooleanDeciderParam(DeciderKey.enableOnboardingNewUserFeatureSet) + + /* + * Param to enable mr user author sparse continuous feature set hydration + * */ + object EnableMrUserAuthorSparseContFeatureSetHydration + extends BooleanDeciderParam(DeciderKey.enableMrUserAuthorSparseContFeatureSet) + + /* + * Param to enable mr user topic sparse continuous feature set hydration + * */ + object EnableMrUserTopicSparseContFeatureSetHydration + extends BooleanDeciderParam(DeciderKey.enableMrUserTopicSparseContFeatureSet) + + /* + * Param to enable penguin language feature set hydration + * */ + object EnableUserPenguinLanguageFeatureSetHydration + extends BooleanDeciderParam(DeciderKey.enableUserPenguinLanguageFeatureSet) + + /* + * Param to enable user engaged tweet tokens feature hydration + * */ + object EnableMrUserEngagedTweetTokensFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableMrUserEngagedTweetTokensFeaturesHydration) + + /* + * Param to enable candidate tweet tokens feature hydration + * */ + object EnableMrCandidateTweetTokensFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableMrCandidateTweetTokensFeaturesHydration) + + /* + * Param to enable mr user hashspace embedding feature set hydration + * */ + object EnableMrUserHashspaceEmbeddingFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableMrUserHashspaceEmbeddingFeatureSet) + + /* + * Param to enable mr tweet sentiment feature set hydration + * */ + object EnableMrTweetSentimentFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableMrTweetSentimentFeatureSet) + + /* + * Param to enable mr tweet_author aggregates feature set hydration + * */ + object EnableMrTweetAuthorAggregatesFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableMrTweetAuthorAggregatesFeatureSet) + + /** + * Param to enable twistly aggregated features + */ + object EnableTwistlyAggregatesFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableTwistlyAggregatesFeatureHydration) + + /** + * Param to enable tweet twhin favoriate features + */ + object EnableTweetTwHINFavFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableTweetTwHINFavFeaturesHydration) + + /* + * Param to enable mr user geo feature set hydration + * */ + object EnableUserGeoFeatureSetHydration + extends BooleanDeciderParam(DeciderKey.enableUserGeoFeatureSet) + + /* + * Param to enable mr author geo feature set hydration + * */ + object EnableAuthorGeoFeatureSetHydration + extends BooleanDeciderParam(DeciderKey.enableAuthorGeoFeatureSet) + + /* + * Param to ramp up mr user geo feature set hydration + * */ + object RampupUserGeoFeatureSetHydration + extends BooleanDeciderParam(DeciderKey.rampupUserGeoFeatureSet) + + /* + * Param to ramp up mr author geo feature set hydration + * */ + object RampupAuthorGeoFeatureSetHydration + extends BooleanDeciderParam(DeciderKey.rampupAuthorGeoFeatureSet) + + /* + * Decider controlled param to enable Pop Geo Tweets + * */ + object PopGeoCandidatesDecider extends BooleanDeciderParam(DeciderKey.enablePopGeoTweets) + + /** + * Decider controlled param to enable Trip Geo Tweets + */ + object TripGeoTweetCandidatesDecider + extends BooleanDeciderParam(DeciderKey.enableTripGeoTweetCandidates) + + /** + * Decider controlled param to enable ContentRecommenderMixerAdaptor + */ + object ContentRecommenderMixerAdaptorDecider + extends BooleanDeciderParam(DeciderKey.enableContentRecommenderMixerAdaptor) + + /** + * Decider controlled param to enable GenericCandidateAdaptor + */ + object GenericCandidateAdaptorDecider + extends BooleanDeciderParam(DeciderKey.enableGenericCandidateAdaptor) + + /** + * Decider controlled param to enable dark traffic to ContentMixer for Trip Geo Tweets + */ + object TripGeoTweetContentMixerDarkTrafficDecider + extends BooleanDeciderParam(DeciderKey.enableTripGeoTweetContentMixerDarkTraffic) + + /* + * Decider controlled param to enable Pop Geo Tweets + * */ + object TrendsCandidateDecider extends BooleanDeciderParam(DeciderKey.enableTrendsTweets) + + /* + * Decider controlled param to enable INS Traffic + **/ + object EnableInsTrafficDecider extends BooleanDeciderParam(DeciderKey.enableInsTraffic) + + /** + * Param to enable assigning pushcap with ML predictions (read from MH table). + * Disabling will fallback to only use heuristics and default values. + */ + object EnableModelBasedPushcapAssignments + extends BooleanDeciderParam(DeciderKey.enableModelBasedPushcapAssignments) + + /** + * Param to enable twhin user engagement feature hydration + */ + object EnableTwHINUserEngagementFeaturesHydration + extends BooleanDeciderParam(DeciderKey.enableTwHINUserEngagementFeaturesHydration) + + /** + * Param to enable twhin user follow feature hydration + */ + object EnableTwHINUserFollowFeaturesHydration + extends BooleanDeciderParam(DeciderKey.enableTwHINUserFollowFeaturesHydration) + + /** + * Param to enable twhin author follow feature hydration + */ + object EnableTwHINAuthorFollowFeaturesHydration + extends BooleanDeciderParam(DeciderKey.enableTwHINAuthorFollowFeaturesHydration) + + /** + * Param to enable calls to the IsTweetTranslatable strato column + */ + object EnableIsTweetTranslatableCheck + extends BooleanDeciderParam(DeciderKey.enableIsTweetTranslatable) + + /** + * Decider controlled param to enable mr tweet simcluster feature set hydration + */ + object EnableMrTweetSimClusterFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableMrTweetSimClusterFeatureSet) + + /** + * Decider controlled param to enable real graph v2 feature set hydration + */ + object EnableRealGraphV2FeatureHydration + extends BooleanDeciderParam(DeciderKey.enableRealGraphV2FeatureHydration) + + /** + * Decider controlled param to enable Tweet BeT feature set hydration + */ + object EnableTweetBeTFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableTweetBeTFeatureHydration) + + /** + * Decider controlled param to enable mr user tweet topic feature set hydration + */ + object EnableMrOfflineUserTweetTopicAggregateHydration + extends BooleanDeciderParam(DeciderKey.enableMrOfflineUserTweetTopicAggregate) + + /** + * Decider controlled param to enable mr tweet simcluster feature set hydration + */ + object EnableMrOfflineUserTweetSimClusterAggregateHydration + extends BooleanDeciderParam(DeciderKey.enableMrOfflineUserTweetSimClusterAggregate) + + /** + * Decider controlled param to enable user send time features + */ + object EnableUserSendTimeFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableUserSendTimeFeatureHydration) + + /** + * Decider controlled param to enable mr user utc send time aggregate features + */ + object EnableMrUserUtcSendTimeAggregateFeaturesHydration + extends BooleanDeciderParam(DeciderKey.enableMrUserUtcSendTimeAggregateFeaturesHydration) + + /** + * Decider controlled param to enable mr user local send time aggregate features + */ + object EnableMrUserLocalSendTimeAggregateFeaturesHydration + extends BooleanDeciderParam(DeciderKey.enableMrUserLocalSendTimeAggregateFeaturesHydration) + + /** + * Decider controlled param to enable BQML report model predictions for F1 tweets + */ + object EnableBqmlReportModelPredictionForF1Tweets + extends BooleanDeciderParam(DeciderKey.enableBqmlReportModelPredictionForF1Tweets) + + /** + * Decider controlled param to enable user Twhin embedding feature hydration + */ + object EnableUserTwhinEmbeddingFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableUserTwhinEmbeddingFeatureHydration) + + /** + * Decider controlled param to enable author follow Twhin embedding feature hydration + */ + object EnableAuthorFollowTwhinEmbeddingFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableAuthorFollowTwhinEmbeddingFeatureHydration) + + object EnableScribingMLFeaturesAsDataRecord + extends BooleanDeciderParam(DeciderKey.enableScribingMLFeaturesAsDataRecord) + + /** + * Decider controlled param to enable feature hydration for Verified related feature + */ + object EnableAuthorVerifiedFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableAuthorVerifiedFeatureHydration) + + /** + * Decider controlled param to enable feature hydration for creator subscription related feature + */ + object EnableAuthorCreatorSubscriptionFeatureHydration + extends BooleanDeciderParam(DeciderKey.enableAuthorCreatorSubscriptionFeatureHydration) + + /** + * Decider controlled param to direct MH+Memcache hydration for the UserFeaturesDataset + */ + object EnableDirectHydrationForUserFeatures + extends BooleanDeciderParam(DeciderKey.enableDirectHydrationForUserFeatures) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.scala new file mode 100644 index 0000000000..7920bb6cdd --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/PushServiceTunableKeys.scala @@ -0,0 +1,9 @@ +package com.twitter.frigate.pushservice.params + +import com.twitter.util.tunable.TunableMap + +object PushServiceTunableKeys { + final val IbisQpsLimitTunableKey = TunableMap.Key[Int]("ibis2.qps.limit") + final val NtabQpsLimitTunableKey = TunableMap.Key[Int]("ntab.qps.limit") + final val TweetPerspectiveStoreQpsLimit = TunableMap.Key[Int]("tweetperspective.qps.limit") +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.scala new file mode 100644 index 0000000000..c0a68c9390 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/params/ShardParams.scala @@ -0,0 +1,3 @@ +package com.twitter.frigate.pushservice.params + +case class ShardParams(numShards: Int, shardId: Int) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.scala new file mode 100644 index 0000000000..67a117cc5f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BigFilteringEpsilonGreedyExplorationPredicate.scala @@ -0,0 +1,58 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.tracing.Trace +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.hashing.KeyHasher +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +/* + * A predicate for epsilon-greedy exploration; + * We defined it as a candidate level predicate to avoid changing the predicate and scribing pipeline, + * but it is actually a post-ranking target level predicate: + * if a target user IS ENABLED for \epsilon-greedy exploration, + * then with probability epsilon, the user (and thus all candidates) will be blocked + */ +object BigFilteringEpsilonGreedyExplorationPredicate { + + val name = "BigFilteringEpsilonGreedyExplorationPredicate" + + private def shouldFilterBasedOnEpsilonGreedyExploration( + target: Target + ): Boolean = { + val seed = KeyHasher.FNV1A_64.hashKey(s"${target.targetId}".getBytes("UTF8")) + val hashKey = KeyHasher.FNV1A_64 + .hashKey( + s"${Trace.id.traceId.toString}:${seed.toString}".getBytes("UTF8") + ) + + math.abs(hashKey).toDouble / Long.MaxValue < + target.params(PushFeatureSwitchParams.MrRequestScribingEpsGreedyExplorationRatio) + } + + def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { + val stats = statsReceiver.scope(s"predicate_$name") + + val enabledForEpsilonGreedyCounter = stats.counter("enabled_for_eps_greedy") + + new Predicate[PushCandidate] { + def apply(candidates: Seq[PushCandidate]): Future[Seq[Boolean]] = { + val results = candidates.map { candidate => + if (!candidate.target.skipFilters && candidate.target.params( + PushFeatureSwitchParams.EnableMrRequestScribingForEpsGreedyExploration)) { + enabledForEpsilonGreedyCounter.incr() + !shouldFilterBasedOnEpsilonGreedyExploration(candidate.target) + } else { + true + } + } + Future.value(results) + } + }.withStats(stats) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.scala new file mode 100644 index 0000000000..f7ff95c9b5 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlHealthModelPredicates.scala @@ -0,0 +1,129 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.ml.HealthFeatureGetter +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.params.PushMLModel +import com.twitter.util.Future +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.frigate.thriftscala.UserMediaRepresentation +import com.twitter.hss.api.thriftscala.UserHealthSignalResponse +import com.twitter.storehaus.ReadableStore + +object BqmlHealthModelPredicates { + + def healthModelOonPredicate( + bqmlHealthModelScorer: PushMLModelScorer, + producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation], + userHealthScoreStore: ReadableStore[Long, UserHealthSignalResponse], + tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] + )( + implicit stats: StatsReceiver + ): NamedPredicate[ + PushCandidate with TweetCandidate with RecommendationType with TweetAuthor + ] = { + val name = "bqml_health_model_based_predicate" + val scopedStatsReceiver = stats.scope(name) + + val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") + val oonCandidatesCounter = scopedStatsReceiver.counter("oon_candidates") + val filteredOonCandidatesCounter = + scopedStatsReceiver.counter("filtered_oon_candidates") + val emptyScoreCandidatesCounter = scopedStatsReceiver.counter("empty_score_candidates") + val healthScoreStat = scopedStatsReceiver.stat("health_model_dist") + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => + val target = candidate.target + val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) || + RecTypes.outOfNetworkTopicTweetTypes.contains(candidate.commonRecType) + + lazy val enableBqmlHealthModelPredicateParam = + target.params(PushFeatureSwitchParams.EnableBqmlHealthModelPredicateParam) + lazy val enableBqmlHealthModelPredictionForInNetworkCandidates = + target.params( + PushFeatureSwitchParams.EnableBqmlHealthModelPredictionForInNetworkCandidatesParam) + lazy val bqmlHealthModelPredicateFilterThresholdParam = + target.params(PushFeatureSwitchParams.BqmlHealthModelPredicateFilterThresholdParam) + lazy val healthModelId = target.params(PushFeatureSwitchParams.BqmlHealthModelTypeParam) + lazy val enableBqmlHealthModelScoreHistogramParam = + target.params(PushFeatureSwitchParams.EnableBqmlHealthModelScoreHistogramParam) + val healthModelScoreFeature = "bqml_health_model_score" + + val histogramBinSize = 0.05 + lazy val healthCandidateScoreHistogramCounters = + bqmlHealthModelScorer.getScoreHistogramCounters( + scopedStatsReceiver, + "health_score_histogram", + histogramBinSize) + + candidate match { + case candidate: PushCandidate with TweetAuthor with TweetAuthorDetails + if enableBqmlHealthModelPredicateParam && (isOonCandidate || enableBqmlHealthModelPredictionForInNetworkCandidates) => + HealthFeatureGetter + .getFeatures( + candidate, + producerMediaRepresentationStore, + userHealthScoreStore, + Some(tweetHealthScoreStore)) + .flatMap { healthFeatures => + allCandidatesCounter.incr() + candidate.mergeFeatures(healthFeatures) + + val healthModelScoreFutOpt = + if (candidate.numericFeatures.contains(healthModelScoreFeature)) { + Future.value(candidate.numericFeatures.get(healthModelScoreFeature)) + } else + bqmlHealthModelScorer.singlePredicationForModelVersion( + healthModelId, + candidate + ) + + candidate.populateQualityModelScore( + PushMLModel.HealthNsfwProbability, + healthModelId, + healthModelScoreFutOpt + ) + + healthModelScoreFutOpt.map { + case Some(healthModelScore) => + healthScoreStat.add((healthModelScore * 10000).toFloat) + if (enableBqmlHealthModelScoreHistogramParam) { + healthCandidateScoreHistogramCounters( + math.ceil(healthModelScore / histogramBinSize).toInt).incr() + } + + if (CandidateUtil.shouldApplyHealthQualityFilters( + candidate) && isOonCandidate) { + oonCandidatesCounter.incr() + val threshold = bqmlHealthModelPredicateFilterThresholdParam + candidate.cachePredicateInfo( + name, + healthModelScore, + threshold, + healthModelScore > threshold) + if (healthModelScore > threshold) { + filteredOonCandidatesCounter.incr() + false + } else true + } else true + case _ => + emptyScoreCandidatesCounter.incr() + true + } + } + case _ => Future.True + } + } + .withStats(stats.scope(name)) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.scala new file mode 100644 index 0000000000..76d52992bc --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/BqmlQualityModelPredicates.scala @@ -0,0 +1,141 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.params.PushConstants.TweetMediaEmbeddingBQKeyIds +import com.twitter.frigate.pushservice.params.PushMLModel +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.util.Future +import com.twitter.frigate.pushservice.util.CandidateUtil._ + +object BqmlQualityModelPredicates { + + def ingestExtraFeatures(cand: PushCandidate): Unit = { + val tagsCRCountFeature = "tagsCR_count" + val hasPushOpenOrNtabClickFeature = "has_PushOpenOrNtabClick" + val onlyPushOpenOrNtabClickFeature = "only_PushOpenOrNtabClick" + val firstTweetMediaEmbeddingFeature = "media_embedding_0" + val tweetMediaEmbeddingFeature = + "media.mediaunderstanding.media_embeddings.twitter_clip_as_sparse_continuous_feature" + + if (!cand.numericFeatures.contains(tagsCRCountFeature)) { + cand.numericFeatures(tagsCRCountFeature) = getTagsCRCount(cand) + } + if (!cand.booleanFeatures.contains(hasPushOpenOrNtabClickFeature)) { + cand.booleanFeatures(hasPushOpenOrNtabClickFeature) = isRelatedToMrTwistlyCandidate(cand) + } + if (!cand.booleanFeatures.contains(onlyPushOpenOrNtabClickFeature)) { + cand.booleanFeatures(onlyPushOpenOrNtabClickFeature) = isMrTwistlyCandidate(cand) + } + if (!cand.numericFeatures.contains(firstTweetMediaEmbeddingFeature)) { + val tweetMediaEmbedding = cand.sparseContinuousFeatures + .getOrElse(tweetMediaEmbeddingFeature, Map.empty[String, Double]) + Seq.range(0, TweetMediaEmbeddingBQKeyIds.size).foreach { i => + cand.numericFeatures(s"media_embedding_$i") = + tweetMediaEmbedding.getOrElse(TweetMediaEmbeddingBQKeyIds(i).toString, 0.0) + } + } + } + + def BqmlQualityModelOonPredicate( + bqmlQualityModelScorer: PushMLModelScorer + )( + implicit stats: StatsReceiver + ): NamedPredicate[ + PushCandidate with TweetCandidate with RecommendationType + ] = { + + val name = "bqml_quality_model_based_predicate" + val scopedStatsReceiver = stats.scope(name) + val oonCandidatesCounter = scopedStatsReceiver.counter("oon_candidates") + val inCandidatesCounter = scopedStatsReceiver.counter("in_candidates") + val filteredOonCandidatesCounter = + scopedStatsReceiver.counter("filtered_oon_candidates") + val bucketedCandidatesCounter = scopedStatsReceiver.counter("bucketed_oon_candidates") + val emptyScoreCandidatesCounter = scopedStatsReceiver.counter("empty_score_candidates") + val histogramBinSize = 0.05 + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => + val target = candidate.target + val crt = candidate.commonRecType + val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || + RecTypes.outOfNetworkTopicTweetTypes.contains(crt) + + lazy val enableBqmlQualityModelScoreHistogramParam = + target.params(PushFeatureSwitchParams.EnableBqmlQualityModelScoreHistogramParam) + + lazy val qualityCandidateScoreHistogramCounters = + bqmlQualityModelScorer.getScoreHistogramCounters( + scopedStatsReceiver, + "quality_score_histogram", + histogramBinSize) + + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && (isOonCandidate || target + .params(PushParams.EnableBqmlReportModelPredictionForF1Tweets)) + && target.params(PushFeatureSwitchParams.EnableBqmlQualityModelPredicateParam)) { + ingestExtraFeatures(candidate) + + lazy val shouldFilterFutSeq = + target + .params(PushFeatureSwitchParams.BqmlQualityModelBucketModelIdListParam) + .zip(target.params(PushFeatureSwitchParams.BqmlQualityModelBucketThresholdListParam)) + .map { + case (modelId, bucketThreshold) => + val scoreFutOpt = + bqmlQualityModelScorer.singlePredicationForModelVersion(modelId, candidate) + + candidate.populateQualityModelScore( + PushMLModel.FilteringProbability, + modelId, + scoreFutOpt + ) + + if (isOonCandidate) { + oonCandidatesCounter.incr() + scoreFutOpt.map { + case Some(score) => + if (score >= bucketThreshold) { + bucketedCandidatesCounter.incr() + if (modelId == target.params( + PushFeatureSwitchParams.BqmlQualityModelTypeParam)) { + if (enableBqmlQualityModelScoreHistogramParam) { + val scoreHistogramBinId = + math.ceil(score / histogramBinSize).toInt + qualityCandidateScoreHistogramCounters(scoreHistogramBinId).incr() + } + if (score >= target.params( + PushFeatureSwitchParams.BqmlQualityModelPredicateThresholdParam)) { + filteredOonCandidatesCounter.incr() + true + } else false + } else false + } else false + case _ => + emptyScoreCandidatesCounter.incr() + false + } + } else { + inCandidatesCounter.incr() + Future.False + } + } + + Future.collect(shouldFilterFutSeq).flatMap { shouldFilterSeq => + if (shouldFilterSeq.contains(true)) { + Future.False + } else Future.True + } + } else Future.True + } + .withStats(stats.scope(name)) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.scala new file mode 100644 index 0000000000..8ccccd14d1 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CaretFeedbackHistoryFilter.scala @@ -0,0 +1,99 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.frigate.common.base.TargetUser +import com.twitter.frigate.common.candidate.CaretFeedbackHistory +import com.twitter.frigate.common.candidate.TargetABDecider +import com.twitter.frigate.common.util.MrNtabCopyObjects +import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails +import com.twitter.notificationservice.thriftscala.GenericNotificationMetadata +import com.twitter.notificationservice.thriftscala.GenericType + +object CaretFeedbackHistoryFilter { + + def caretFeedbackHistoryFilter( + categories: Seq[String] + ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ + CaretFeedbackDetails + ] = { target => caretFeedbackDetailsSeq => + caretFeedbackDetailsSeq.filter { caretFeedbackDetails => + caretFeedbackDetails.genericNotificationMetadata match { + case Some(genericNotificationMetadata) => + isFeedbackSupportedGenericType(genericNotificationMetadata) + case None => false + } + } + } + + private def filterCriteria( + caretFeedbackDetails: CaretFeedbackDetails, + genericTypes: Seq[GenericType] + ): Boolean = { + caretFeedbackDetails.genericNotificationMetadata match { + case Some(genericNotificationMetadata) => + genericTypes.contains(genericNotificationMetadata.genericType) + case None => false + } + } + + def caretFeedbackHistoryFilterByGenericType( + genericTypes: Seq[GenericType] + ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ + CaretFeedbackDetails + ] = { target => caretFeedbackDetailsSeq => + caretFeedbackDetailsSeq.filter { caretFeedbackDetails => + filterCriteria(caretFeedbackDetails, genericTypes) + } + } + + def caretFeedbackHistoryFilterByGenericTypeDenyList( + genericTypes: Seq[GenericType] + ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ + CaretFeedbackDetails + ] = { target => caretFeedbackDetailsSeq => + caretFeedbackDetailsSeq.filterNot { caretFeedbackDetails => + filterCriteria(caretFeedbackDetails, genericTypes) + } + } + + def caretFeedbackHistoryFilterByRefreshableType( + refreshableTypes: Set[Option[String]] + ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ + CaretFeedbackDetails + ] = { target => caretFeedbackDetailsSeq => + caretFeedbackDetailsSeq.filter { caretFeedbackDetails => + caretFeedbackDetails.genericNotificationMetadata match { + case Some(genericNotificationMetadata) => + refreshableTypes.contains(genericNotificationMetadata.refreshableType) + case None => false + } + } + } + + def caretFeedbackHistoryFilterByRefreshableTypeDenyList( + refreshableTypes: Set[Option[String]] + ): TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[CaretFeedbackDetails] => Seq[ + CaretFeedbackDetails + ] = { target => caretFeedbackDetailsSeq => + caretFeedbackDetailsSeq.filter { caretFeedbackDetails => + caretFeedbackDetails.genericNotificationMetadata match { + case Some(genericNotificationMetadata) => + !refreshableTypes.contains(genericNotificationMetadata.refreshableType) + case None => true + } + } + } + + private def isFeedbackSupportedGenericType( + notificationMetadata: GenericNotificationMetadata + ): Boolean = { + val genericNotificationTypeName = + (notificationMetadata.genericType, notificationMetadata.refreshableType) match { + case (GenericType.RefreshableNotification, Some(refreshableType)) => refreshableType + case _ => notificationMetadata.genericType.name + } + + MrNtabCopyObjects.AllNtabCopyTypes + .flatMap(_.refreshableType) + .contains(genericNotificationTypeName) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.scala new file mode 100644 index 0000000000..22067405af --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CasLockPredicate.scala @@ -0,0 +1,45 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.util.CasLock +import com.twitter.frigate.common.util.CasSuccess +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Duration +import com.twitter.util.Future + +object CasLockPredicate { + def apply( + casLock: CasLock, + expiryDuration: Duration + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val stats = statsReceiver.scope("predicate_addcaslock_for_candidate") + Predicate + .fromAsync { candidate: PushCandidate => + if (candidate.target.pushContext.exists(_.darkWrite.exists(_ == true))) { + Future.True + } else if (candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent) { + Future.True + } else { + candidate.target.history flatMap { h => + val now = candidate.createdAt + val expiry = now + expiryDuration + val oldTimestamp = h.lastNotificationTime map { + _.inSeconds + } getOrElse 0 + casLock.cas(candidate.target.targetId, oldTimestamp, now.inSeconds, expiry) map { + casResult => + stats.counter(s"cas_$casResult").incr() + casResult == CasSuccess + } + } + } + } + .withStats(stats) + .withName("add_cas_lock") + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.scala new file mode 100644 index 0000000000..4b1abf221a --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/CrtDeciderPredicate.scala @@ -0,0 +1,25 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.decider.Decider +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate + +object CrtDeciderPredicate { + val name = "crt_decider" + def apply( + decider: Decider + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate] = { + Predicate + .from { (candidate: PushCandidate) => + val prefix = "frigate_pushservice_" + val deciderKey = prefix + candidate.commonRecType + decider.feature(deciderKey).isAvailable + } + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.scala new file mode 100644 index 0000000000..cb55be3564 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/DiscoverTwitterPredicate.scala @@ -0,0 +1,47 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TargetUser +import com.twitter.frigate.common.candidate.FrigateHistory +import com.twitter.frigate.common.history.History +import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate +import com.twitter.frigate.common.predicate.{FatiguePredicate => TargetFatiguePredicate} +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.hermit.predicate.Predicate +import com.twitter.timelines.configapi.Param +import com.twitter.util.Duration + +object DiscoverTwitterPredicate { + + /** + * Predicate used to determine if a minimum duration has elapsed since the last MR push + * for a CRT to be valid. + * @param name Identifier of the caller (used for stats) + * @param intervalParam The minimum duration interval + * @param stats StatsReceiver + * @return Target Predicate + */ + def minDurationElapsedSinceLastMrPushPredicate( + name: String, + intervalParam: Param[Duration], + stats: StatsReceiver + ): Predicate[Target] = + Predicate + .fromAsync { target: Target => + val interval = + target.params(intervalParam) + FrigateHistoryFatiguePredicate( + minInterval = interval, + getSortedHistory = { h: History => + val magicRecsOnlyHistory = + TargetFatiguePredicate.magicRecsPushOnlyFilter(h.sortedPushDmHistory) + TargetFatiguePredicate.magicRecsNewUserPlaybookPushFilter(magicRecsOnlyHistory) + } + ).flatContraMap { target: TargetUser with FrigateHistory => + target.history + }.apply(Seq(target)).map { + _.head + } + }.withStats(stats.scope(s"${name}_predicate_mr_push_min_interval")) + .withName(s"${name}_predicate_mr_push_min_interval") +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.scala new file mode 100644 index 0000000000..457dc879c3 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/FatiguePredicate.scala @@ -0,0 +1,74 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.predicate.FatiguePredicate._ +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.{NotificationDisplayLocation => DisplayLocation} +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.util.Duration + +object FatiguePredicate { + + /** + * Predicate that operates on a candidate, and applies custom fatigue rules for the slice of history only + * corresponding to a given rec type. + * + * @param interval + * @param maxInInterval + * @param minInterval + * @param recommendationType + * @param statsReceiver + * @return + */ + def recTypeOnly( + interval: Duration, + maxInInterval: Int, + minInterval: Duration, + recommendationType: CommonRecommendationType, + notificationDisplayLocation: DisplayLocation = DisplayLocation.PushToMobileDevice + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate] = { + build( + interval = interval, + maxInInterval = maxInInterval, + minInterval = minInterval, + filterHistory = recOnlyFilter(recommendationType), + notificationDisplayLocation = notificationDisplayLocation + ).flatContraMap { candidate: PushCandidate => candidate.target.history } + .withStats(statsReceiver.scope(s"predicate_${recTypeOnlyFatigue}")) + .withName(recTypeOnlyFatigue) + } + + /** + * Predicate that operates on a candidate, and applies custom fatigue rules for the slice of history only + * corresponding to specified rec types + * + * @param interval + * @param maxInInterval + * @param minInterval + * @param statsReceiver + * @return + */ + def recTypeSetOnly( + interval: Duration, + maxInInterval: Int, + minInterval: Duration, + recTypes: Set[CommonRecommendationType], + notificationDisplayLocation: DisplayLocation = DisplayLocation.PushToMobileDevice + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "rec_type_set_fatigue" + build( + interval = interval, + maxInInterval = maxInInterval, + minInterval = minInterval, + filterHistory = recTypesOnlyFilter(recTypes), + notificationDisplayLocation = notificationDisplayLocation + ).flatContraMap { candidate: PushCandidate => candidate.target.history } + .withStats(statsReceiver.scope(s"${name}_predicate")) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.scala new file mode 100644 index 0000000000..f11ed1400a --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/HealthPredicates.scala @@ -0,0 +1,740 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse +import com.twitter.abuse.detection.scoring.thriftscala.{Model => TweetHealthModel} +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.NsfwTextDetectionModel +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.util.CandidateHydrationUtil +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil +import com.twitter.frigate.thriftscala.UserMediaRepresentation +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.hss.api.thriftscala.UserHealthSignal._ +import com.twitter.hss.api.thriftscala.SignalValue +import com.twitter.hss.api.thriftscala.UserHealthSignalResponse +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future +import com.twitter.util.Time + +object HealthPredicates { + + private val NsfwTextDetectionModelMap: Map[NsfwTextDetectionModel.Value, TweetHealthModel] = + Map( + NsfwTextDetectionModel.ProdModel -> TweetHealthModel.PnsfwTweetText, + NsfwTextDetectionModel.RetrainedModel -> TweetHealthModel.ExperimentalHealthModelScore1, + ) + + private def tweetIsSupportedLanguage( + candidate: PushCandidate, + supportedLanguages: Set[String] + ): Boolean = { + val tweetLanguage = + candidate.categoricalFeatures.getOrElse("RecTweet.TweetyPieResult.Language", "") + supportedLanguages.contains(tweetLanguage) + } + + def tweetHealthSignalScorePredicate( + tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse], + applyToQuoteTweet: Boolean = false + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate with TweetDetails] = { + val name = "tweet_health_signal_store_applyToQuoteTweet_" + applyToQuoteTweet.toString + val scopedStatsReceiver = stats.scope(name) + val numCandidatesStats = scopedStatsReceiver.scope("num_candidates") + val numCandidatesMediaNsfwScoreStats = numCandidatesStats.scope("media_nsfw_score") + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate with TweetDetails => + numCandidatesStats.counter("all").incr() + val target = candidate.target + val tweetIdOpt = if (!applyToQuoteTweet) { + Some(candidate.tweetId) + } else candidate.tweetyPieResult.flatMap(_.quotedTweet.map(_.id)) + + tweetIdOpt match { + case Some(tweetId) => + val pMediaNsfwRequest = + TweetScoringRequest(tweetId, TweetHealthModel.ExperimentalHealthModelScore4) + tweetHealthScoreStore.get(pMediaNsfwRequest).map { + case Some(tweetScoringResponse) => + numCandidatesMediaNsfwScoreStats.counter("non_empty").incr() + val pMediaNsfwScore = tweetScoringResponse.score + + if (!applyToQuoteTweet) { + candidate + .cacheExternalScore("NsfwMediaProbability", Future.value(Some(pMediaNsfwScore))) + } + + val pMediaNsfwShouldBucket = + pMediaNsfwScore > target.params( + PushFeatureSwitchParams.PnsfwTweetMediaBucketingThreshold) + if (CandidateUtil.shouldApplyHealthQualityFilters( + candidate) && pMediaNsfwShouldBucket) { + numCandidatesMediaNsfwScoreStats.counter("bucketed").incr() + if (target.params(PushFeatureSwitchParams.PnsfwTweetMediaFilterOonOnly) + && !RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType)) { + true + } else { + val pMediaNsfwScoreThreshold = + if (applyToQuoteTweet) + target.params(PushFeatureSwitchParams.PnsfwQuoteTweetThreshold) + else if (candidate.hasPhoto) + target.params(PushFeatureSwitchParams.PnsfwTweetImageThreshold) + else target.params(PushFeatureSwitchParams.PnsfwTweetMediaThreshold) + candidate.cachePredicateInfo( + name + "_nsfwMedia", + pMediaNsfwScore, + pMediaNsfwScoreThreshold, + pMediaNsfwScore > pMediaNsfwScoreThreshold) + if (pMediaNsfwScore > pMediaNsfwScoreThreshold) { + numCandidatesMediaNsfwScoreStats.counter("filtered").incr() + false + } else true + } + } else true + case _ => + numCandidatesMediaNsfwScoreStats.counter("empty").incr() + if (candidate.hasPhoto || candidate.hasVideo) { + numCandidatesMediaNsfwScoreStats.counter("media_tweet_with_empty_score").incr() + } + true + } + case _ => Future.True + } + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } + + def healthSignalScoreSpammyTweetPredicate( + tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate with TweetDetails] = { + val name = "health_signal_store_spammy_tweet" + val statsScope = stats.scope(name) + val allCandidatesCounter = statsScope.counter("all_candidates") + val eligibleCandidatesCounter = statsScope.counter("eligible_candidates") + val oonCandidatesCounter = statsScope.counter("oon_candidates") + val inCandidatesCounter = statsScope.counter("in_candidates") + val bucketedCandidatesCounter = statsScope.counter("num_bucketed") + val nonEmptySpamScoreCounter = statsScope.counter("non_empty_spam_score") + val filteredOonCandidatesCounter = statsScope.counter("num_filtered_oon") + val filteredInCandidatesCounter = statsScope.counter("num_filtered_in") + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate with TweetDetails => + allCandidatesCounter.incr() + val crt = candidate.commonRecType + val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || + RecTypes.outOfNetworkTopicTweetTypes.contains(crt) + if (isOonCandidate) { + oonCandidatesCounter.incr() + } + val target = candidate.target + if (target.params(PushFeatureSwitchParams.EnableSpammyTweetFilter)) { + eligibleCandidatesCounter.incr() + val tweetSpamScore = + TweetScoringRequest(candidate.tweetId, TweetHealthModel.SpammyTweetContent) + tweetHealthScoreStore.get(tweetSpamScore).map { + case (Some(tweetScoringResponse)) => + nonEmptySpamScoreCounter.incr() + val candidateSpamScore = tweetScoringResponse.score + + candidate + .cacheExternalScore("SpammyTweetScore", Future.value(Some(candidateSpamScore))) + + val tweetSpamShouldBucket = + candidateSpamScore > target.params( + PushFeatureSwitchParams.SpammyTweetBucketingThreshold) + if (CandidateUtil.shouldApplyHealthQualityFilters( + candidate) && tweetSpamShouldBucket) { + bucketedCandidatesCounter.incr() + if (isOonCandidate) { + val spamScoreThreshold = + target.params(PushFeatureSwitchParams.SpammyTweetOonThreshold) + if (candidateSpamScore > spamScoreThreshold) { + filteredOonCandidatesCounter.incr() + false + } else true + } else { + inCandidatesCounter.incr() + val spamScoreThreshold = + target.params(PushFeatureSwitchParams.SpammyTweetInThreshold) + if (candidateSpamScore > spamScoreThreshold) { + filteredInCandidatesCounter.incr() + false + } else true + } + } else true + case _ => true + } + } else Future.True + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } + + def healthSignalScorePnsfwTweetTextPredicate( + tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate] = { + val name = "health_signal_store_pnsfw_tweet_text" + val statsScope = stats.scope(name) + val allCandidatesCounter = statsScope.counter("all_candidates") + val nonEmptyNsfwTextScoreNum = statsScope.counter("non_empty_nsfw_text_score") + val filteredCounter = statsScope.counter("num_filtered") + val lowScoreCounter = statsScope.counter("low_score_count") + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate => + val target = candidate.target + val predEnabled = + target.params(PushFeatureSwitchParams.EnableHealthSignalStorePnsfwTweetTextPredicate) + if (CandidateUtil.shouldApplyHealthQualityFilters( + candidate) && predEnabled && tweetIsSupportedLanguage(candidate, Set(""))) { + allCandidatesCounter.incr() + val pnsfwTextRequest = + TweetScoringRequest(candidate.tweetId, TweetHealthModel.PnsfwTweetText) + tweetHealthScoreStore.get(pnsfwTextRequest).flatMap { + case Some(tweetScoringResponse) => { + nonEmptyNsfwTextScoreNum.incr() + if (tweetScoringResponse.score < 1e-8) { + lowScoreCounter.incr() + } + + candidate + .cacheExternalScore( + "NsfwTextProbability-en", + Future.value(Some(tweetScoringResponse.score))) + val threshold = target.params(PushFeatureSwitchParams.PnsfwTweetTextThreshold) + candidate.cachePredicateInfo( + name, + tweetScoringResponse.score, + threshold, + tweetScoringResponse.score > threshold) + if (tweetScoringResponse.score > threshold) { + filteredCounter.incr() + Future.False + } else Future.True + } + case _ => Future.True + } + } else Future.True + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } + + def healthSignalScoreMultilingualPnsfwTweetTextPredicate( + tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate] = { + val name = "health_signal_store_multilingual_pnsfw_tweet_text" + val statsScope = stats.scope(name) + + val allLanguagesIdentifier = "all" + val languagesSelectedForStats = + Set("") + allLanguagesIdentifier + + val candidatesCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => + lang -> statsScope.counter(f"candidates_$lang") + }.toMap + val nonEmptyHealthScoreMap: Map[String, Counter] = languagesSelectedForStats.map { lang => + lang -> statsScope.counter(f"non_empty_health_score_$lang") + }.toMap + val emptyHealthScoreMap: Map[String, Counter] = languagesSelectedForStats.map { lang => + lang -> statsScope.counter(f"empty_health_score_$lang") + }.toMap + val bucketedCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => + lang -> statsScope.counter(f"num_candidates_bucketed_$lang") + }.toMap + val filteredCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => + lang -> statsScope.counter(f"num_filtered_$lang") + }.toMap + val lowScoreCounterMap: Map[String, Counter] = languagesSelectedForStats.map { lang => + lang -> statsScope.counter(f"low_score_count_$lang") + }.toMap + + val wrongBucketingModelCounter = statsScope.counter("wrong_bucketing_model_count") + val wrongDetectionModelCounter = statsScope.counter("wrong_detection_model_count") + + def increaseCounterForLanguage(counterMap: Map[String, Counter], language: String): Unit = { + counterMap.get(allLanguagesIdentifier) match { + case Some(counter) => counter.incr() + case _ => + } + counterMap.get(language) match { + case Some(counter) => counter.incr() + case _ => + } + } + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate => + val target = candidate.target + + val languageFeatureName = "RecTweet.TweetyPieResult.Language" + + lazy val isPredicateEnabledForTarget = target.params( + PushFeatureSwitchParams.EnableHealthSignalStoreMultilingualPnsfwTweetTextPredicate) + + lazy val targetNsfwTextDetectionModel: NsfwTextDetectionModel.Value = + target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextModel) + + lazy val targetPredicateSupportedLanguageSeq: Seq[String] = + target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextSupportedLanguages) + + lazy val bucketingModelSeq: Seq[NsfwTextDetectionModel.Value] = + target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextBucketingModelList) + + lazy val bucketingThresholdPerLanguageSeq: Seq[Double] = + target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextBucketingThreshold) + + lazy val filteringThresholdPerLanguageSeq: Seq[Double] = + target.params(PushFeatureSwitchParams.MultilingualPnsfwTweetTextFilteringThreshold) + + if (CandidateUtil.shouldApplyHealthQualityFilters( + candidate) && isPredicateEnabledForTarget) { + val candidateLanguage = + candidate.categoricalFeatures.getOrElse(languageFeatureName, "") + + val indexOfCandidateLanguage = + targetPredicateSupportedLanguageSeq.indexOf(candidateLanguage) + + val isCandidateLanguageSupported = indexOfCandidateLanguage >= 0 + + if (isCandidateLanguageSupported) { + increaseCounterForLanguage(candidatesCounterMap, candidateLanguage) + + val bucketingModelScoreMap: Map[NsfwTextDetectionModel.Value, Future[Option[Double]]] = + bucketingModelSeq.map { modelName => + NsfwTextDetectionModelMap.get(modelName) match { + case Some(targetNsfwTextDetectionModel) => + val pnsfwTweetTextRequest: TweetScoringRequest = + TweetScoringRequest(candidate.tweetId, targetNsfwTextDetectionModel) + + val scoreOptFut: Future[Option[Double]] = + tweetHealthScoreStore.get(pnsfwTweetTextRequest).map(_.map(_.score)) + + candidate + .cacheExternalScore("NsfwTextProbability", scoreOptFut) + + modelName -> scoreOptFut + case _ => + wrongBucketingModelCounter.incr() + modelName -> Future.None + } + }.toMap + + val candidateLanguageBucketingThreshold = + bucketingThresholdPerLanguageSeq(indexOfCandidateLanguage) + + val userShouldBeBucketedFut: Future[Boolean] = + Future + .collect(bucketingModelScoreMap.map { + case (_, modelScoreOptFut) => + modelScoreOptFut.map { + case Some(score) => + increaseCounterForLanguage(nonEmptyHealthScoreMap, candidateLanguage) + score > candidateLanguageBucketingThreshold + case _ => + increaseCounterForLanguage(emptyHealthScoreMap, candidateLanguage) + false + } + }.toSeq).map(_.contains(true)) + + val candidateShouldBeFilteredFut: Future[Boolean] = userShouldBeBucketedFut.flatMap { + userShouldBeBucketed => + if (userShouldBeBucketed) { + increaseCounterForLanguage(bucketedCounterMap, candidateLanguage) + + val candidateLanguageFilteringThreshold = + filteringThresholdPerLanguageSeq(indexOfCandidateLanguage) + + bucketingModelScoreMap.get(targetNsfwTextDetectionModel) match { + case Some(scoreOptFut) => + scoreOptFut.map { + case Some(score) => + val candidateShouldBeFiltered = + score > candidateLanguageFilteringThreshold + if (candidateShouldBeFiltered) { + increaseCounterForLanguage(filteredCounterMap, candidateLanguage) + } + candidateShouldBeFiltered + case _ => false + } + case _ => + wrongDetectionModelCounter.incr() + Future.False + } + } else { + increaseCounterForLanguage(lowScoreCounterMap, candidateLanguage) + Future.False + } + } + candidateShouldBeFilteredFut.map(result => !result) + } else Future.True + } else Future.True + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } + + def authorProfileBasedPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate] = { + val name = "author_profile" + val statsScope = stats.scope(name) + val filterByNsfwToken = statsScope.counter("filter_by_nsfw_token") + val filterByAccountAge = statsScope.counter("filter_by_account_age") + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate => + val target = candidate.target + candidate match { + case cand: PushCandidate with TweetAuthorDetails => + cand.tweetAuthor.map { + case Some(author) => + val nsfwTokens = target.params(PushFeatureSwitchParams.NsfwTokensParam) + val accountAgeInHours = + (Time.now - Time.fromMilliseconds(author.createdAtMsec)).inHours + val isNsfwAccount = CandidateHydrationUtil.isNsfwAccount(author, nsfwTokens) + val isVerified = author.safety.map(_.verified).getOrElse(false) + + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && !isVerified) { + val enableNsfwTokenCheck = + target.params(PushFeatureSwitchParams.EnableNsfwTokenBasedFiltering) + val minimumAllowedAge = + target.params(PushFeatureSwitchParams.MinimumAllowedAuthorAccountAgeInHours) + cand.cachePredicateInfo( + name + "_nsfwToken", + if (isNsfwAccount) 1.0 else 0.0, + 0.0, + enableNsfwTokenCheck && isNsfwAccount) + cand.cachePredicateInfo( + name + "_authorAge", + accountAgeInHours, + minimumAllowedAge, + accountAgeInHours < minimumAllowedAge) + + if (enableNsfwTokenCheck && isNsfwAccount) { + filterByNsfwToken.incr() + false + } else if (accountAgeInHours < minimumAllowedAge) { + filterByAccountAge.incr() + false + } else true + } else true + case _ => true + } + case _ => Future.value(true) + } + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } + + def authorSensitiveMediaPredicate( + producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor] = { + val name = "author_sensitive_media_mrtwistly" + val statsScope = stats.scope(name) + val enableQueryNum = statsScope.counter("enable_query") + val nonEmptyMediaRepresentationNum = statsScope.counter("non_empty_media_representation") + val filteredOON = statsScope.counter("filtered_oon") + + Predicate + .fromAsync { candidate: PushCandidate with TweetAuthor => + val target = candidate.target + val useAggressiveThresholds = CandidateUtil.useAggressiveHealthThresholds(candidate) + + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && + RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) && + target.params(PushFeatureSwitchParams.EnableQueryAuthorMediaRepresentationStore)) { + enableQueryNum.incr() + + candidate.authorId match { + case Some(authorId) => + producerMediaRepresentationStore.get(authorId).map { + case Some(mediaRepresentation) => + nonEmptyMediaRepresentationNum.incr() + val sumScore: Double = mediaRepresentation.mediaRepresentation.values.sum + val nudityScore: Double = mediaRepresentation.mediaRepresentation + .getOrElse(MediaAnnotationsUtil.nudityCategoryId, 0.0) + val nudityRate = if (sumScore > 0) nudityScore / sumScore else 0.0 + + candidate + .cacheExternalScore("AuthorNudityScore", Future.value(Some(nudityScore))) + candidate.cacheExternalScore("AuthorNudityRate", Future.value(Some(nudityRate))) + + val threshold = if (useAggressiveThresholds) { + target.params( + PushFeatureSwitchParams.AuthorSensitiveMediaFilteringThresholdForMrTwistly) + } else { + target.params(PushFeatureSwitchParams.AuthorSensitiveMediaFilteringThreshold) + } + candidate.cachePredicateInfo( + name, + nudityRate, + threshold, + nudityRate > threshold, + Some(Map[String, Double]("sumScore" -> sumScore, "nudityScore" -> nudityScore))) + + if (nudityRate > threshold) { + filteredOON.incr() + false + } else true + case _ => true + } + case _ => Future.True + } + } else { + Future.True + } + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } + + def sensitiveMediaCategoryPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate] = { + val name = "sensitive_media_category" + val tweetMediaAnnotationFeature = + "tweet.mediaunderstanding.tweet_annotations.sensitive_category_probabilities" + val scopedStatsReceiver = stats.scope(name) + val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") + val nonZeroNudityCandidatesCounter = scopedStatsReceiver.counter("non_zero_nudity_candidates") + val nudityScoreStats = scopedStatsReceiver.stat("nudity_scores") + + Predicate + .fromAsync { candidate: PushCandidate => + allCandidatesCounter.incr() + val target = candidate.target + val nudityScore = candidate.sparseContinuousFeatures + .getOrElse(tweetMediaAnnotationFeature, Map.empty[String, Double]).getOrElse( + MediaAnnotationsUtil.nudityCategoryId, + 0.0) + if (nudityScore > 0) nonZeroNudityCandidatesCounter.incr() + nudityScoreStats.add(nudityScore.toFloat) + val threshold = + target.params(PushFeatureSwitchParams.TweetMediaSensitiveCategoryThresholdParam) + candidate.cachePredicateInfo(name, nudityScore, threshold, nudityScore > threshold) + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && nudityScore > threshold) { + Future.False + } else { + Future.True + } + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } + + def profanityPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate] = { + val name = "profanity_filter" + val scopedStatsReceiver = stats.scope(name) + val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") + + Predicate + .fromAsync { candidate: PushCandidate => + allCandidatesCounter.incr() + val target = candidate.target + + lazy val enableFilter = + target.params(PushFeatureSwitchParams.EnableProfanityFilterParam) + val tweetSemanticCoreIds = candidate.sparseBinaryFeatures + .getOrElse(PushConstants.TweetSemanticCoreIdFeature, Set.empty[String]) + + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && + tweetSemanticCoreIds.contains(PushConstants.ProfanityFilter_Id) && enableFilter) { + Future.False + } else { + Future.True + } + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } + + def agathaAbusiveTweetAuthorPredicateMrTwistly( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with OutOfNetworkTweetCandidate] = { + val name = "agatha_abusive_tweet_author_mr_twistly" + val scopedStatsReceiver = stats.scope(name) + val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") + val isMrBackfillCRCandidateCounter = scopedStatsReceiver.counter("isMrBackfillCR_candidates") + Predicate + .fromAsync { cand: PushCandidate with OutOfNetworkTweetCandidate => + allCandidatesCounter.incr() + val target = cand.target + val tweetSemanticCoreIds = cand.sparseBinaryFeatures + .getOrElse(PushConstants.TweetSemanticCoreIdFeature, Set.empty[String]) + + val hasAbuseStrikeTop2Percent = + tweetSemanticCoreIds.contains(PushConstants.AbuseStrike_Top2Percent_Id) + val hasAbuseStrikeTop1Percent = + tweetSemanticCoreIds.contains(PushConstants.AbuseStrike_Top1Percent_Id) + val hasAbuseStrikeTop05Percent = + tweetSemanticCoreIds.contains(PushConstants.AbuseStrike_Top05Percent_Id) + + if (hasAbuseStrikeTop2Percent) { + scopedStatsReceiver.counter("abuse_strike_top_2_percent_candidates").incr() + } + if (hasAbuseStrikeTop1Percent) { + scopedStatsReceiver.counter("abuse_strike_top_1_percent_candidates").incr() + } + if (hasAbuseStrikeTop05Percent) { + scopedStatsReceiver.counter("abuse_strike_top_05_percent_candidates").incr() + } + + if (CandidateUtil.shouldApplyHealthQualityFilters(cand) && cand.isMrBackfillCR.getOrElse( + false)) { + isMrBackfillCRCandidateCounter.incr() + if (hasAbuseStrikeTop2Percent) { + if (target.params( + PushFeatureSwitchParams.EnableAbuseStrikeTop2PercentFilterSimCluster) && hasAbuseStrikeTop2Percent || + target.params( + PushFeatureSwitchParams.EnableAbuseStrikeTop1PercentFilterSimCluster) && hasAbuseStrikeTop1Percent || + target.params( + PushFeatureSwitchParams.EnableAbuseStrikeTop05PercentFilterSimCluster) && hasAbuseStrikeTop05Percent) { + Future.False + } else { + Future.True + } + } else { + Future.True + } + } else Future.True + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } + + def userHealthSignalsPredicate( + userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse] + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with TweetDetails] = { + val name = "agatha_user_health_model_score" + val scopedStatsReceiver = stats.scope(name) + val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") + val bucketedUserCandidatesCounter = + scopedStatsReceiver.counter("bucketed_user_candidates") + val filteredOON = scopedStatsReceiver.counter("filtered_oon") + + Predicate + .fromAsync { candidate: PushCandidate with TweetDetails => + allCandidatesCounter.incr() + val target = candidate.target + val useAggressiveThresholds = CandidateUtil.useAggressiveHealthThresholds(candidate) + + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && target.params( + PushFeatureSwitchParams.EnableAgathaUserHealthModelPredicate)) { + val healthSignalsResponseFutOpt: Future[Option[UserHealthSignalResponse]] = + candidate.authorId match { + case Some(authorId) => userHealthSignalStore.get(authorId) + case _ => Future.None + } + healthSignalsResponseFutOpt.map { + case Some(response) => + val agathaRecentAbuseStrikeScore: Double = userHealthSignalValueToDouble( + response.signalValues + .getOrElse(AgathaRecentAbuseStrikeDouble, SignalValue.DoubleValue(0.0))) + val agathaCalibratedNSFWScore: Double = userHealthSignalValueToDouble( + response.signalValues + .getOrElse(AgathaCalibratedNsfwDouble, SignalValue.DoubleValue(0.0))) + val agathaTextNSFWScore: Double = userHealthSignalValueToDouble(response.signalValues + .getOrElse(NsfwTextUserScoreDouble, SignalValue.DoubleValue(0.0))) + + candidate + .cacheExternalScore( + "agathaRecentAbuseStrikeScore", + Future.value(Some(agathaRecentAbuseStrikeScore))) + candidate + .cacheExternalScore( + "agathaCalibratedNSFWScore", + Future.value(Some(agathaCalibratedNSFWScore))) + candidate + .cacheExternalScore("agathaTextNSFWScore", Future.value(Some(agathaTextNSFWScore))) + + val NSFWShouldBucket = agathaCalibratedNSFWScore > target.params( + PushFeatureSwitchParams.AgathaCalibratedNSFWBucketThreshold) + val textNSFWShouldBucket = agathaTextNSFWScore > target.params( + PushFeatureSwitchParams.AgathaTextNSFWBucketThreshold) + + if (NSFWShouldBucket || textNSFWShouldBucket) { + bucketedUserCandidatesCounter.incr() + if (NSFWShouldBucket) { + scopedStatsReceiver.counter("calibrated_nsfw_bucketed_user_candidates").incr() + } + if (textNSFWShouldBucket) { + scopedStatsReceiver.counter("text_nsfw_bucketed_user_candidates").incr() + } + + val (thresholdAgathaNsfw, thresholdTextNsfw) = if (useAggressiveThresholds) { + ( + target.params( + PushFeatureSwitchParams.AgathaCalibratedNSFWThresholdForMrTwistly), + target + .params(PushFeatureSwitchParams.AgathaTextNSFWThresholdForMrTwistly)) + } else { + ( + target.params(PushFeatureSwitchParams.AgathaCalibratedNSFWThreshold), + target.params(PushFeatureSwitchParams.AgathaTextNSFWThreshold)) + } + candidate.cachePredicateInfo( + name + "_agathaNsfw", + agathaCalibratedNSFWScore, + thresholdAgathaNsfw, + agathaCalibratedNSFWScore > thresholdAgathaNsfw) + candidate.cachePredicateInfo( + name + "_authorTextNsfw", + agathaTextNSFWScore, + thresholdTextNsfw, + agathaTextNSFWScore > thresholdTextNsfw) + + if ((agathaCalibratedNSFWScore > thresholdAgathaNsfw) || + (agathaTextNSFWScore > thresholdTextNsfw)) { + filteredOON.incr() + false + } else true + } else { + true + } + case _ => true + } + } else { + Future.True + } + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } + + def userHealthSignalValueToDouble(signalValue: SignalValue): Double = { + signalValue match { + case SignalValue.DoubleValue(value) => value + case _ => throw new Exception(f"Could not convert signal value to double") + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.scala new file mode 100644 index 0000000000..63095f4db1 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/JointDauAndQualityModelPredicate.scala @@ -0,0 +1,39 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.QualityPredicateIdParam +import com.twitter.frigate.pushservice.predicate.quality_model_predicate._ +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +object JointDauAndQualityModelPredicate { + + val name = "JointDauAndQualityModelPredicate" + + def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { + val stats = statsReceiver.scope(s"predicate_$name") + + val defaultPred = WeightedOpenOrNtabClickQualityPredicate() + val qualityPredicateMap = QualityPredicateMap() + + Predicate + .fromAsync { candidate: PushCandidate => + if (!candidate.target.skipModelPredicate) { + + val modelPredicate = + qualityPredicateMap.getOrElse( + candidate.target.params(QualityPredicateIdParam), + defaultPred) + + val modelPredicateResultFut = + modelPredicate.apply(Seq(candidate)).map(_.headOption.getOrElse(false)) + + modelPredicateResultFut + } else Future.True + } + .withStats(stats) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.scala new file mode 100644 index 0000000000..cbfb670d8f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ListPredicates.scala @@ -0,0 +1,110 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.model.ListRecommendationPushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.hermit.predicate.socialgraph.Edge +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.socialgraph.thriftscala.RelationshipType +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +object ListPredicates { + + def listNameExistsPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[ListRecommendationPushCandidate] = { + Predicate + .fromAsync { candidate: ListRecommendationPushCandidate => + candidate.listName.map(_.isDefined) + } + .withStats(stats) + .withName("list_name_exists") + } + + def listAuthorExistsPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[ListRecommendationPushCandidate] = { + Predicate + .fromAsync { candidate: ListRecommendationPushCandidate => + candidate.listOwnerId.map(_.isDefined) + } + .withStats(stats) + .withName("list_owner_exists") + } + + def listAuthorAcceptableToTargetUser( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[ListRecommendationPushCandidate] = { + val name = "list_author_acceptable_to_target_user" + val sgsPredicate = SocialGraphPredicate + .anyRelationExists( + edgeStore, + Set( + RelationshipType.Blocking, + RelationshipType.BlockedBy, + RelationshipType.Muting + ) + ) + .withStats(statsReceiver.scope("list_sgs_any_relation_exists")) + .withName("list_sgs_any_relation_exists") + + Predicate + .fromAsync { candidate: ListRecommendationPushCandidate => + candidate.listOwnerId.flatMap { + case Some(ownerId) => + sgsPredicate.apply(Seq(Edge(candidate.target.targetId, ownerId))).map(_.head) + case _ => Future.True + } + } + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } + + /** + * Checks if the list is acceptable to Target user => + * - Is Target not following the list + * - Is Target not muted the list + */ + def listAcceptablePredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[ListRecommendationPushCandidate] = { + val name = "list_acceptable_to_target_user" + Predicate + .fromAsync { candidate: ListRecommendationPushCandidate => + candidate.apiList.map { + case Some(apiList) => + !(apiList.following.contains(true) || apiList.muting.contains(true)) + case _ => false + } + } + .withStats(stats.scope(name)) + .withName(name) + } + + def listSubscriberCountPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[ListRecommendationPushCandidate] = { + val name = "list_subscribe_count" + Predicate + .fromAsync { candidate: ListRecommendationPushCandidate => + candidate.apiList.map { apiListOpt => + apiListOpt.exists { apiList => + apiList.subscriberCount >= candidate.target.params( + PushFeatureSwitchParams.ListRecommendationsSubscriberCount) + } + } + } + .withStats(stats.scope(name)) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.scala new file mode 100644 index 0000000000..9ba1c9f6f8 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutPreRankingPredicates.scala @@ -0,0 +1,37 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.base.TweetDetails +import com.twitter.frigate.common.predicate.tweet._ +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.predicate.NamedPredicate + +class LoggedOutPreRankingPredicatesBuilder(implicit statsReceiver: StatsReceiver) { + + private val TweetPredicates = List[NamedPredicate[PushCandidate]]( + TweetObjectExistsPredicate[ + TweetCandidate with TweetDetails + ].applyOnlyToTweetCandidatesWithTweetDetails + .withName("tweet_object_exists"), + PredicatesForCandidate.oldTweetRecsPredicate.applyOnlyToTweetCandidateWithTargetAndABDeciderAndMaxTweetAge + .withName("old_tweet"), + PredicatesForCandidate.tweetIsNotAreply.applyOnlyToTweetCandidateWithoutSocialContextWithTweetDetails + .withName("tweet_candidate_not_a_reply"), + TweetAuthorPredicates + .recTweetAuthorUnsuitable[TweetCandidate with TweetAuthorDetails] + .applyOnlyToTweetCandidateWithTweetAuthorDetails + .withName("tweet_author_unsuitable") + ) + + final def build(): List[NamedPredicate[PushCandidate]] = { + TweetPredicates + } + +} + +object LoggedOutPreRankingPredicates { + def apply(statsReceiver: StatsReceiver): List[NamedPredicate[PushCandidate]] = + new LoggedOutPreRankingPredicatesBuilder()(statsReceiver).build() +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.scala new file mode 100644 index 0000000000..085ad73e91 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/LoggedOutTargetPredicates.scala @@ -0,0 +1,53 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.abdecider.GuestRecipient +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.common.predicate.{FatiguePredicate => CommonFatiguePredicate} +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.conversions.DurationOps._ +import com.twitter.frigate.common.util.Experiments.LoggedOutRecsHoldback +import com.twitter.hermit.predicate.Predicate + +object LoggedOutTargetPredicates { + + def targetFatiguePredicate[T <: Target]( + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + val name = "logged_out_target_min_duration_since_push" + CommonFatiguePredicate + .magicRecsPushTargetFatiguePredicate( + minInterval = 24.hours, + maxInInterval = 1 + ).withStats(statsReceiver.scope(name)) + .withName(name) + } + + def loggedOutRecsHoldbackPredicate[T <: Target]( + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + val name = "logged_out_recs_holdback" + val guestIdNotFoundCounter = statsReceiver.scope("logged_out").counter("guest_id_not_found") + val controlBucketCounter = statsReceiver.scope("logged_out").counter("holdback_control") + val allowTrafficCounter = statsReceiver.scope("logged_out").counter("allow_traffic") + Predicate.from { target: T => + val guestId = target.targetGuestId match { + case Some(guest) => guest + case _ => + guestIdNotFoundCounter.incr() + throw new IllegalStateException("guest_id_not_found") + } + target.abDecider + .bucket(LoggedOutRecsHoldback.exptName, GuestRecipient(guestId)).map(_.name) match { + case Some(LoggedOutRecsHoldback.control) => + controlBucketCounter.incr() + false + case _ => + allowTrafficCounter.incr() + true + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.scala new file mode 100644 index 0000000000..014393870c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/MlModelsHoldbackExperimentPredicate.scala @@ -0,0 +1,71 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +object MlModelsHoldbackExperimentPredicate { + + val name = "MlModelsHoldbackExperimentPredicate" + + private val alwaysTruePred = PredicatesForCandidate.alwaysTruePushCandidatePredicate + + def getPredicateBasedOnCandidate( + pc: PushCandidate, + treatmentPred: Predicate[PushCandidate] + )( + implicit statsReceiver: StatsReceiver + ): Future[Predicate[PushCandidate]] = { + + Future + .join(Future.value(pc.target.skipFilters), pc.target.isInModelExclusionList) + .map { + case (skipFilters, isInModelExclusionList) => + if (skipFilters || + isInModelExclusionList || + pc.target.params(PushParams.DisableMlInFilteringParam) || + pc.target.params(PushFeatureSwitchParams.DisableMlInFilteringFeatureSwitchParam) || + pc.target.params(PushParams.DisableAllRelevanceParam) || + pc.target.params(PushParams.DisableHeavyRankingParam)) { + alwaysTruePred + } else { + treatmentPred + } + } + } + + def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { + val stats = statsReceiver.scope(s"predicate_$name") + val statsProd = stats.scope("prod") + val counterAcceptedByModel = statsProd.counter("accepted") + val counterRejectedByModel = statsProd.counter("rejected") + val counterHoldback = stats.scope("holdback").counter("all") + val jointDauQualityPredicate = JointDauAndQualityModelPredicate() + + new Predicate[PushCandidate] { + def apply(items: Seq[PushCandidate]): Future[Seq[Boolean]] = { + val boolFuts = items.map { item => + getPredicateBasedOnCandidate(item, jointDauQualityPredicate)(statsReceiver) + .flatMap { predicate => + val predictionFut = predicate.apply(Seq(item)).map(_.headOption.getOrElse(false)) + predictionFut.foreach { prediction => + if (item.target.params(PushParams.DisableMlInFilteringParam) || item.target.params( + PushFeatureSwitchParams.DisableMlInFilteringFeatureSwitchParam)) { + counterHoldback.incr() + } else { + if (prediction) counterAcceptedByModel.incr() else counterRejectedByModel.incr() + } + } + predictionFut + } + } + Future.collect(boolFuts) + } + }.withStats(stats) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.scala new file mode 100644 index 0000000000..bcd9e30d07 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONSpreadControlPredicate.scala @@ -0,0 +1,116 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushConstants._ +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +object OONSpreadControlPredicate { + + def oonTweetSpreadControlPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[ + PushCandidate with TweetCandidate with RecommendationType + ] = { + val name = "oon_tweet_spread_control_predicate" + val scopedStatsReceiver = stats.scope(name) + val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") + val filteredCandidatesCounter = + scopedStatsReceiver.counter("filtered_oon_candidates") + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => + val target = candidate.target + val crt = candidate.commonRecType + val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || + RecTypes.outOfNetworkTopicTweetTypes.contains(crt) + + lazy val minTweetSendsThreshold = + target.params(PushFeatureSwitchParams.MinTweetSendsThresholdParam) + lazy val spreadControlRatio = + target.params(PushFeatureSwitchParams.SpreadControlRatioParam) + lazy val favOverSendThreshold = + target.params(PushFeatureSwitchParams.FavOverSendThresholdParam) + + lazy val sentCount = candidate.numericFeatures.getOrElse(sentFeatureName, 0.0) + lazy val followerCount = + candidate.numericFeatures.getOrElse(authorActiveFollowerFeatureName, 0.0) + lazy val favCount = candidate.numericFeatures.getOrElse(favFeatureName, 0.0) + lazy val favOverSends = favCount / (sentCount + 1.0) + + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { + allOonCandidatesCounter.incr() + if (sentCount > minTweetSendsThreshold && + sentCount > spreadControlRatio * followerCount && + favOverSends < favOverSendThreshold) { + filteredCandidatesCounter.incr() + Future.False + } else Future.True + } else Future.True + } + .withStats(stats.scope(name)) + .withName(name) + } + + def oonAuthorSpreadControlPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[ + PushCandidate with TweetCandidate with RecommendationType + ] = { + val name = "oon_author_spread_control_predicate" + val scopedStatsReceiver = stats.scope(name) + val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") + val filteredCandidatesCounter = + scopedStatsReceiver.counter("filtered_oon_candidates") + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => + val target = candidate.target + val crt = candidate.commonRecType + val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || + RecTypes.outOfNetworkTopicTweetTypes.contains(crt) + + lazy val minAuthorSendsThreshold = + target.params(PushFeatureSwitchParams.MinAuthorSendsThresholdParam) + lazy val spreadControlRatio = + target.params(PushFeatureSwitchParams.SpreadControlRatioParam) + lazy val reportRateThreshold = + target.params(PushFeatureSwitchParams.AuthorReportRateThresholdParam) + lazy val dislikeRateThreshold = + target.params(PushFeatureSwitchParams.AuthorDislikeRateThresholdParam) + + lazy val authorSentCount = + candidate.numericFeatures.getOrElse(authorSendCountFeatureName, 0.0) + lazy val authorReportCount = + candidate.numericFeatures.getOrElse(authorReportCountFeatureName, 0.0) + lazy val authorDislikeCount = + candidate.numericFeatures.getOrElse(authorDislikeCountFeatureName, 0.0) + lazy val followerCount = candidate.numericFeatures + .getOrElse(authorActiveFollowerFeatureName, 0.0) + lazy val reportRate = + authorReportCount / (authorSentCount + 1.0) + lazy val dislikeRate = + authorDislikeCount / (authorSentCount + 1.0) + + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { + allOonCandidatesCounter.incr() + if (authorSentCount > minAuthorSendsThreshold && + authorSentCount > spreadControlRatio * followerCount && + (reportRate > reportRateThreshold || dislikeRate > dislikeRateThreshold)) { + filteredCandidatesCounter.incr() + Future.False + } else Future.True + } else Future.True + } + .withStats(stats.scope(name)) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.scala new file mode 100644 index 0000000000..3efb23d88f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OONTweetNegativeFeedbackBasedPredicate.scala @@ -0,0 +1,82 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +object OONTweetNegativeFeedbackBasedPredicate { + + def ntabDislikeBasedPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[ + PushCandidate with TweetCandidate with RecommendationType + ] = { + val name = "oon_tweet_dislike_based_predicate" + val scopedStatsReceiver = stats.scope(name) + val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") + val oonCandidatesImpressedCounter = + scopedStatsReceiver.counter("oon_candidates_impressed") + val filteredCandidatesCounter = + scopedStatsReceiver.counter("filtered_oon_candidates") + + val ntabDislikeCountFeature = + "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_ntab_disliked.any_feature.Duration.Top.count" + val sentFeature = + "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_sent.any_feature.Duration.Top.count" + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => + val target = candidate.target + val crt = candidate.commonRecType + val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || + RecTypes.outOfNetworkTopicTweetTypes.contains(crt) + + lazy val ntabDislikeCountThreshold = + target.params(PushFeatureSwitchParams.TweetNtabDislikeCountThresholdParam) + lazy val ntabDislikeRateThreshold = + target.params(PushFeatureSwitchParams.TweetNtabDislikeRateThresholdParam) + lazy val ntabDislikeCountThresholdForMrTwistly = + target.params(PushFeatureSwitchParams.TweetNtabDislikeCountThresholdForMrTwistlyParam) + lazy val ntabDislikeRateThresholdForMrTwistly = + target.params(PushFeatureSwitchParams.TweetNtabDislikeRateThresholdForMrTwistlyParam) + + val isMrTwistly = CandidateUtil.isMrTwistlyCandidate(candidate) + + lazy val dislikeCount = candidate.numericFeatures.getOrElse(ntabDislikeCountFeature, 0.0) + lazy val sentCount = candidate.numericFeatures.getOrElse(sentFeature, 0.0) + lazy val dislikeRate = if (sentCount > 0) dislikeCount / sentCount else 0.0 + + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { + allOonCandidatesCounter.incr() + val (countThreshold, rateThreshold) = if (isMrTwistly) { + (ntabDislikeCountThresholdForMrTwistly, ntabDislikeRateThresholdForMrTwistly) + } else { + (ntabDislikeCountThreshold, ntabDislikeRateThreshold) + } + candidate.cachePredicateInfo( + name + "_count", + dislikeCount, + countThreshold, + dislikeCount > countThreshold) + candidate.cachePredicateInfo( + name + "_rate", + dislikeRate, + rateThreshold, + dislikeRate > rateThreshold) + if (dislikeCount > countThreshold && dislikeRate > rateThreshold) { + filteredCandidatesCounter.incr() + Future.False + } else Future.True + } else Future.True + } + .withStats(stats.scope(name)) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.scala new file mode 100644 index 0000000000..6f09df0c72 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/OutOfNetworkCandidatesQualityPredicates.scala @@ -0,0 +1,221 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.candidate.TargetABDecider +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient +import com.twitter.util.Future +import com.twitter.frigate.pushservice.predicate.PostRankingPredicateHelper._ +import com.twitter.frigate.pushservice.util.CandidateUtil + +object OutOfNetworkCandidatesQualityPredicates { + + def getTweetCharLengthThreshold( + target: TargetUser with TargetABDecider, + language: String, + useMediaThresholds: Boolean + ): Double = { + lazy val sautOonWithMediaTweetLengthThreshold = + target.params(PushFeatureSwitchParams.SautOonWithMediaTweetLengthThresholdParam) + lazy val nonSautOonWithMediaTweetLengthThreshold = + target.params(PushFeatureSwitchParams.NonSautOonWithMediaTweetLengthThresholdParam) + lazy val sautOonWithoutMediaTweetLengthThreshold = + target.params(PushFeatureSwitchParams.SautOonWithoutMediaTweetLengthThresholdParam) + lazy val nonSautOonWithoutMediaTweetLengthThreshold = + target.params(PushFeatureSwitchParams.NonSautOonWithoutMediaTweetLengthThresholdParam) + val moreStrictForUndefinedLanguages = + target.params(PushFeatureSwitchParams.OonTweetLengthPredicateMoreStrictForUndefinedLanguages) + val isSautLanguage = if (moreStrictForUndefinedLanguages) { + isTweetLanguageInSautOrUndefined(language) + } else isTweetLanguageInSaut(language) + + (useMediaThresholds, isSautLanguage) match { + case (true, true) => + sautOonWithMediaTweetLengthThreshold + case (true, false) => + nonSautOonWithMediaTweetLengthThreshold + case (false, true) => + sautOonWithoutMediaTweetLengthThreshold + case (false, false) => + nonSautOonWithoutMediaTweetLengthThreshold + case _ => -1 + } + } + + def getTweetWordLengthThreshold( + target: TargetUser with TargetABDecider, + language: String, + useMediaThresholds: Boolean + ): Double = { + lazy val argfOonWithMediaTweetWordLengthThresholdParam = + target.params(PushFeatureSwitchParams.ArgfOonWithMediaTweetWordLengthThresholdParam) + lazy val esfthOonWithMediaTweetWordLengthThresholdParam = + target.params(PushFeatureSwitchParams.EsfthOonWithMediaTweetWordLengthThresholdParam) + + lazy val argfOonCandidatesWithMediaCondition = + isTweetLanguageInArgf(language) && useMediaThresholds + lazy val esfthOonCandidatesWithMediaCondition = + isTweetLanguageInEsfth(language) && useMediaThresholds + lazy val afirfOonCandidatesWithoutMediaCondition = + isTweetLanguageInAfirf(language) && !useMediaThresholds + + val afirfOonCandidatesWithoutMediaTweetWordLengthThreshold = 5 + if (argfOonCandidatesWithMediaCondition) { + argfOonWithMediaTweetWordLengthThresholdParam + } else if (esfthOonCandidatesWithMediaCondition) { + esfthOonWithMediaTweetWordLengthThresholdParam + } else if (afirfOonCandidatesWithoutMediaCondition) { + afirfOonCandidatesWithoutMediaTweetWordLengthThreshold + } else -1 + } + + def oonTweetLengthBasedPrerankingPredicate( + characterBased: Boolean + )( + implicit stats: StatsReceiver + ): NamedPredicate[OutOfNetworkTweetCandidate with TargetInfo[ + TargetUser with TargetABDecider + ]] = { + val name = "oon_tweet_length_based_preranking_predicate" + val scopedStats = stats.scope(s"${name}_charBased_$characterBased") + + Predicate + .fromAsync { + cand: OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider] => + cand match { + case candidate: TweetAuthorDetails => + val target = candidate.target + val crt = candidate.commonRecType + + val updatedMediaLogic = + target.params(PushFeatureSwitchParams.OonTweetLengthPredicateUpdatedMediaLogic) + val updatedQuoteTweetLogic = + target.params(PushFeatureSwitchParams.OonTweetLengthPredicateUpdatedQuoteTweetLogic) + val useMediaThresholds = if (updatedMediaLogic || updatedQuoteTweetLogic) { + val hasMedia = updatedMediaLogic && (candidate.hasPhoto || candidate.hasVideo) + val hasQuoteTweet = updatedQuoteTweetLogic && candidate.quotedTweet.nonEmpty + hasMedia || hasQuoteTweet + } else RecTypes.isMediaType(crt) + val enableFilter = + target.params(PushFeatureSwitchParams.EnablePrerankingTweetLengthPredicate) + + val language = candidate.tweet.flatMap(_.language.map(_.language)).getOrElse("") + val tweetTextOpt = candidate.tweet.flatMap(_.coreData.map(_.text)) + + val (length: Double, threshold: Double) = if (characterBased) { + ( + tweetTextOpt.map(_.size.toDouble).getOrElse(9999.0), + getTweetCharLengthThreshold(target, language, useMediaThresholds)) + } else { + ( + tweetTextOpt.map(getTweetWordLength).getOrElse(999.0), + getTweetWordLengthThreshold(target, language, useMediaThresholds)) + } + scopedStats.counter("threshold_" + threshold.toString).incr() + + CandidateUtil.shouldApplyHealthQualityFiltersForPrerankingPredicates(candidate).map { + case true if enableFilter => + length > threshold + case _ => true + } + case _ => + scopedStats.counter("author_is_not_hydrated").incr() + Future.True + } + }.withStats(scopedStats) + .withName(name) + } + + private def isTweetLanguageInAfirf(candidateLanguage: String): Boolean = { + val setAFIRF: Set[String] = Set("") + setAFIRF.contains(candidateLanguage) + } + private def isTweetLanguageInEsfth(candidateLanguage: String): Boolean = { + val setESFTH: Set[String] = Set("") + setESFTH.contains(candidateLanguage) + } + private def isTweetLanguageInArgf(candidateLanguage: String): Boolean = { + val setARGF: Set[String] = Set("") + setARGF.contains(candidateLanguage) + } + + private def isTweetLanguageInSaut(candidateLanguage: String): Boolean = { + val setSAUT = Set("") + setSAUT.contains(candidateLanguage) + } + + private def isTweetLanguageInSautOrUndefined(candidateLanguage: String): Boolean = { + val setSautOrUndefined = Set("") + setSautOrUndefined.contains(candidateLanguage) + } + + def containTargetNegativeKeywords(text: String, denylist: Seq[String]): Boolean = { + if (denylist.isEmpty) + false + else { + denylist + .map { negativeKeyword => + text.toLowerCase().contains(negativeKeyword) + }.reduce(_ || _) + } + } + + def NegativeKeywordsPredicate( + postRankingFeatureStoreClient: DynamicFeatureStoreClient[MrRequestContextForFeatureStore] + )( + implicit stats: StatsReceiver + ): NamedPredicate[ + PushCandidate with TweetCandidate with RecommendationType + ] = { + + val name = "negative_keywords_predicate" + val scopedStatsReceiver = stats.scope(name) + val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") + val filteredOonCandidatesCounter = scopedStatsReceiver.counter("filtered_oon_candidates") + val tweetLanguageFeature = "RecTweet.TweetyPieResult.Language" + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => + val target = candidate.target + val crt = candidate.commonRecType + val isTwistlyCandidate = RecTypes.twistlyTweets.contains(crt) + + lazy val enableNegativeKeywordsPredicateParam = + target.params(PushFeatureSwitchParams.EnableNegativeKeywordsPredicateParam) + lazy val negativeKeywordsPredicateDenylist = + target.params(PushFeatureSwitchParams.NegativeKeywordsPredicateDenylist) + lazy val candidateLanguage = + candidate.categoricalFeatures.getOrElse(tweetLanguageFeature, "") + + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && candidateLanguage.equals( + "en") && isTwistlyCandidate && enableNegativeKeywordsPredicateParam) { + allOonCandidatesCounter.incr() + + val tweetTextFuture: Future[String] = + getTweetText(candidate, postRankingFeatureStoreClient) + + tweetTextFuture.map { tweetText => + val containsNegativeWords = + containTargetNegativeKeywords(tweetText, negativeKeywordsPredicateDenylist) + candidate.cachePredicateInfo( + name, + if (containsNegativeWords) 1.0 else 0.0, + 0.0, + containsNegativeWords) + if (containsNegativeWords) { + filteredOonCandidatesCounter.incr() + false + } else true + } + } else Future.True + } + .withStats(stats.scope(name)) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.scala new file mode 100644 index 0000000000..f838d7ae6e --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PNegMultimodalPredicates.scala @@ -0,0 +1,83 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.abuse.detection.scoring.thriftscala.Model +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringRequest +import com.twitter.abuse.detection.scoring.thriftscala.TweetScoringResponse +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +object PNegMultimodalPredicates { + + def healthSignalScorePNegMultimodalPredicate( + tweetHealthScoreStore: ReadableStore[TweetScoringRequest, TweetScoringResponse] + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate] = { + val name = "pneg_multimodal_predicate" + val statsScope = stats.scope(name) + val oonCandidatesCounter = statsScope.counter("oon_candidates") + val nonEmptyModelScoreCounter = statsScope.counter("non_empty_model_score") + val bucketedCounter = statsScope.counter("bucketed_oon_candidates") + val filteredCounter = statsScope.counter("filtered_oon_candidates") + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate => + val target = candidate.target + val crt = candidate.commonRecType + val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || + RecTypes.outOfNetworkTopicTweetTypes.contains(crt) + + lazy val enablePNegMultimodalPredicateParam = + target.params(PushFeatureSwitchParams.EnablePNegMultimodalPredicateParam) + lazy val pNegMultimodalPredicateModelThresholdParam = + target.params(PushFeatureSwitchParams.PNegMultimodalPredicateModelThresholdParam) + lazy val pNegMultimodalPredicateBucketThresholdParam = + target.params(PushFeatureSwitchParams.PNegMultimodalPredicateBucketThresholdParam) + val pNegMultimodalEnabledForF1Tweets = + target.params(PushParams.EnablePnegMultimodalPredictionForF1Tweets) + + if (CandidateUtil.shouldApplyHealthQualityFilters( + candidate) && (isOonCandidate || pNegMultimodalEnabledForF1Tweets) && enablePNegMultimodalPredicateParam) { + + val pNegMultimodalRequest = TweetScoringRequest(candidate.tweetId, Model.PNegMultimodal) + tweetHealthScoreStore.get(pNegMultimodalRequest).map { + case Some(tweetScoringResponse) => + nonEmptyModelScoreCounter.incr() + + val pNegMultimodalScore = 1.0 - tweetScoringResponse.score + + candidate + .cacheExternalScore("PNegMultimodalScore", Future.value(Some(pNegMultimodalScore))) + + if (isOonCandidate) { + oonCandidatesCounter.incr() + + if (pNegMultimodalScore > pNegMultimodalPredicateBucketThresholdParam) { + bucketedCounter.incr() + if (pNegMultimodalScore > pNegMultimodalPredicateModelThresholdParam) { + filteredCounter.incr() + false + } else true + } else true + } else { + true + } + case _ => true + } + } else { + Future.True + } + } + .withStats(stats.scope(name)) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.scala new file mode 100644 index 0000000000..604f7b07c5 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PostRankingPredicateHelper.scala @@ -0,0 +1,50 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.frigate.common.base._ +import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.ml.featurestore.catalog.entities.core.Tweet +import com.twitter.ml.featurestore.catalog.features.core.Tweet.Text +import com.twitter.ml.featurestore.lib.TweetId +import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient +import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest +import com.twitter.util.Future + +object PostRankingPredicateHelper { + + val tweetTextFeature = "tweet.core.tweet.text" + + def getTweetText( + candidate: PushCandidate with TweetCandidate, + dynamicClient: DynamicFeatureStoreClient[MrRequestContextForFeatureStore] + ): Future[String] = { + if (candidate.categoricalFeatures.contains(tweetTextFeature)) { + Future.value(candidate.categoricalFeatures.getOrElse(tweetTextFeature, "")) + } else { + val candidateTweetEntity = Tweet.withId(TweetId(candidate.tweetId)) + val featureStoreRequests = Seq( + FeatureStoreRequest( + entityIds = Seq(candidateTweetEntity) + )) + val predictionRecords = dynamicClient( + featureStoreRequests, + requestContext = candidate.target.mrRequestContextForFeatureStore) + + predictionRecords.map { records => + val tweetText = records.head + .getFeatureValue(candidateTweetEntity, Text).getOrElse( + "" + ) + candidate.categoricalFeatures(tweetTextFeature) = tweetText + tweetText + } + } + } + + def getTweetWordLength(tweetText: String): Double = { + val tweetTextWithoutUrl: String = + tweetText.replaceAll("https?://\\S+\\s?", "").replaceAll("[\\s]+", " ") + tweetTextWithoutUrl.trim().split(" ").length.toDouble + } + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.scala new file mode 100644 index 0000000000..4b61b23e3f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PreRankingPredicates.scala @@ -0,0 +1,158 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.SocialContextActions +import com.twitter.frigate.common.base.SocialContextUserDetails +import com.twitter.frigate.common.base.TargetInfo +import com.twitter.frigate.common.base.TargetUser +import com.twitter.frigate.common.base.TweetAuthor +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.base.TweetDetails +import com.twitter.frigate.common.candidate.FrigateHistory +import com.twitter.frigate.common.candidate.TargetABDecider +import com.twitter.frigate.common.candidate.TweetImpressionHistory +import com.twitter.frigate.common.predicate.socialcontext.{Predicates => SocialContextPredicates, _} +import com.twitter.frigate.common.predicate.tweet._ +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.NtabCaretClickContFnFatiguePredicate +import com.twitter.hermit.predicate.NamedPredicate + +class PreRankingPredicatesBuilder( +)( + implicit statsReceiver: StatsReceiver) { + + private val SocialProofPredicates = List[NamedPredicate[PushCandidate]]( + SocialContextPredicates + .authorInSocialContext() + .applyOnlyToTweetAuthorWithSocialContextActions + .withName("author_social_context"), + SocialContextPredicates + .selfInSocialContext[TargetUser, SocialContextActions with TargetInfo[TargetUser]]() + .applyOnlyToSocialContextActionsWithTargetUser + .withName("self_social_context"), + SocialContextPredicates + .duplicateSocialContext[SocialContextActions]() + .applyOnlyToSocialContextActions + .withName("duplicate_social_context"), + SocialContextPredicates + .socialContextProtected[SocialContextUserDetails]() + .applyOnlyToSocialContextUserDetails + .withName("social_context_protected"), + SocialContextPredicates + .socialContextUnsuitable[SocialContextUserDetails]() + .applyOnlyToSocialContextUserDetails + .withName("social_context_unsuitable"), + SocialContextPredicates + .socialContextBlink[SocialContextUserDetails]() + .applyOnlyToSocialContextUserDetails + .withName("social_context_blink") + ) + + private val CommonPredicates = List[NamedPredicate[PushCandidate]]( + PredicatesForCandidate.candidateEnabledForEmailPredicate(), + PredicatesForCandidate.openAppExperimentUserCandidateAllowList(statsReceiver) + ) + + private val TweetPredicates = List[NamedPredicate[PushCandidate]]( + PredicatesForCandidate.tweetCandidateWithLessThan2SocialContextsIsAReply.applyOnlyToTweetCandidatesWithSocialContextActions + .withName("tweet_candidate_with_less_than_2_social_contexts_is_not_a_reply"), + PredicatesForCandidate.filterOONCandidatePredicate(), + PredicatesForCandidate.oldTweetRecsPredicate.applyOnlyToTweetCandidateWithTargetAndABDeciderAndMaxTweetAge + .withName("old_tweet"), + DuplicatePushTweetPredicate + .apply[ + TargetUser with FrigateHistory, + TweetCandidate with TargetInfo[TargetUser with FrigateHistory] + ] + .applyOnlyToTweetCandidateWithTargetAndFrigateHistory + .withName("duplicate_push_tweet"), + DuplicateEmailTweetPredicate + .apply[ + TargetUser with FrigateHistory, + TweetCandidate with TargetInfo[TargetUser with FrigateHistory] + ] + .applyOnlyToTweetCandidateWithTargetAndFrigateHistory + .withName("duplicate_email_tweet"), + TweetAuthorPredicates + .recTweetAuthorUnsuitable[TweetCandidate with TweetAuthorDetails] + .applyOnlyToTweetCandidateWithTweetAuthorDetails + .withName("tweet_author_unsuitable"), + TweetObjectExistsPredicate[ + TweetCandidate with TweetDetails + ].applyOnlyToTweetCandidatesWithTweetDetails + .withName("tweet_object_exists"), + TweetImpressionPredicate[ + TargetUser with TweetImpressionHistory, + TweetCandidate with TargetInfo[TargetUser with TweetImpressionHistory] + ].applyOnlyToTweetCandidateWithTargetAndTweetImpressionHistory + .withStats(statsReceiver.scope("tweet_impression")) + .withName("tweet_impression"), + SelfTweetPredicate[ + TargetUser, + TweetAuthor with TargetInfo[TargetUser]]().applyOnlyToTweetAuthorWithTargetInfo + .withName("self_author"), + PredicatesForCandidate.tweetIsNotAreply.applyOnlyToTweetCandidateWithoutSocialContextWithTweetDetails + .withName("tweet_candidate_not_a_reply"), + PredicatesForCandidate.f1CandidateIsNotAReply.applyOnlyToF1CandidateWithTargetAndABDecider + .withName("f1_candidate_is_not_a_reply"), + PredicatesForCandidate.outOfNetworkTweetCandidateIsNotAReply.applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider + .withName("out_of_network_tweet_candidate_is_not_a_reply"), + PredicatesForCandidate.outOfNetworkTweetCandidateEnabledCrTag.applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider + .withName("out_of_network_tweet_candidate_enabled_crtag"), + PredicatesForCandidate.outOfNetworkTweetCandidateEnabledCrtGroup.applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider + .withName("out_of_network_tweet_candidate_enabled_crt_group"), + OutOfNetworkCandidatesQualityPredicates + .oonTweetLengthBasedPrerankingPredicate(characterBased = true) + .applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider + .withName("oon_tweet_char_length_too_short"), + OutOfNetworkCandidatesQualityPredicates + .oonTweetLengthBasedPrerankingPredicate(characterBased = false) + .applyOnlyToOutOfNetworkTweetCandidateWithTargetAndABDecider + .withName("oon_tweet_word_length_too_short"), + PredicatesForCandidate + .protectedTweetF1ExemptPredicate[ + TargetUser with TargetABDecider, + TweetCandidate with TweetAuthorDetails with TargetInfo[ + TargetUser with TargetABDecider + ] + ] + .applyOnlyToTweetCandidateWithAuthorDetailsWithTargetABDecider + .withName("f1_exempt_tweet_author_protected"), + ) + + private val SgsPreRankingPredicates = List[NamedPredicate[PushCandidate]]( + SGSPredicatesForCandidate.authorBeingFollowed.applyOnlyToAuthorBeingFollowPredicates + .withName("author_not_being_followed"), + SGSPredicatesForCandidate.authorNotBeingDeviceFollowed.applyOnlyToBasicTweetPredicates + .withName("author_being_device_followed"), + SGSPredicatesForCandidate.recommendedTweetAuthorAcceptableToTargetUser.applyOnlyToBasicTweetPredicates + .withName("recommended_tweet_author_not_acceptable_to_target_user"), + SGSPredicatesForCandidate.disableInNetworkTweetPredicate.applyOnlyToBasicTweetPredicates + .withName("enable_in_network_tweet"), + SGSPredicatesForCandidate.disableOutNetworkTweetPredicate.applyOnlyToBasicTweetPredicates + .withName("enable_out_network_tweet") + ) + + private val SeeLessOftenPredicates = List[NamedPredicate[PushCandidate]]( + NtabCaretClickContFnFatiguePredicate + .ntabCaretClickContFnFatiguePredicates( + ) + .withName("seelessoften_cont_fn_fatigue") + ) + + final def build(): List[NamedPredicate[PushCandidate]] = { + TweetPredicates ++ + CommonPredicates ++ + SocialProofPredicates ++ + SgsPreRankingPredicates ++ + SeeLessOftenPredicates + } +} + +object PreRankingPredicates { + def apply( + statsReceiver: StatsReceiver + ): List[NamedPredicate[PushCandidate]] = + new PreRankingPredicatesBuilder()(statsReceiver).build() +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.scala new file mode 100644 index 0000000000..e18667b51d --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/PredicatesForCandidate.scala @@ -0,0 +1,874 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.candidate.MaxTweetAge +import com.twitter.frigate.common.candidate.TargetABDecider +import com.twitter.frigate.common.predicate.tweet.TweetAuthorPredicates +import com.twitter.frigate.common.predicate._ +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.common.util.SnowflakeUtils +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.frigate.thriftscala.ChannelName +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.gizmoduck.thriftscala.UserType +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.hermit.predicate.gizmoduck._ +import com.twitter.hermit.predicate.socialgraph.Edge +import com.twitter.hermit.predicate.socialgraph.MultiEdge +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate +import com.twitter.service.metastore.gen.thriftscala.Location +import com.twitter.socialgraph.thriftscala.RelationshipType +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.configapi.Param +import com.twitter.util.Duration +import com.twitter.util.Future + +object PredicatesForCandidate { + + def oldTweetRecsPredicate(implicit stats: StatsReceiver): Predicate[ + TweetCandidate with RecommendationType with TargetInfo[ + TargetUser with TargetABDecider with MaxTweetAge + ] + ] = { + val name = "old_tweet" + Predicate + .from[TweetCandidate with RecommendationType with TargetInfo[ + TargetUser with TargetABDecider with MaxTweetAge + ]] { candidate => + { + val crt = candidate.commonRecType + val defaultAge = if (RecTypes.mrModelingBasedTypes.contains(crt)) { + candidate.target.params(PushFeatureSwitchParams.ModelingBasedCandidateMaxTweetAgeParam) + } else if (RecTypes.GeoPopTweetTypes.contains(crt)) { + candidate.target.params(PushFeatureSwitchParams.GeoPopTweetMaxAgeInHours) + } else if (RecTypes.simclusterBasedTweets.contains(crt)) { + candidate.target.params( + PushFeatureSwitchParams.SimclusterBasedCandidateMaxTweetAgeParam) + } else if (RecTypes.detopicTypes.contains(crt)) { + candidate.target.params(PushFeatureSwitchParams.DetopicBasedCandidateMaxTweetAgeParam) + } else if (RecTypes.f1FirstDegreeTypes.contains(crt)) { + candidate.target.params(PushFeatureSwitchParams.F1CandidateMaxTweetAgeParam) + } else if (crt == CommonRecommendationType.ExploreVideoTweet) { + candidate.target.params(PushFeatureSwitchParams.ExploreVideoTweetAgeParam) + } else + candidate.target.params(PushFeatureSwitchParams.MaxTweetAgeParam) + SnowflakeUtils.isRecent(candidate.tweetId, defaultAge) + } + } + .withStats(stats.scope(name)) + .withName(name) + } + + def tweetIsNotAreply( + implicit stats: StatsReceiver + ): NamedPredicate[TweetCandidate with TweetDetails] = { + val name = "tweet_candidate_not_a_reply" + Predicate + .from[TweetCandidate with TweetDetails] { c => + c.isReply match { + case Some(true) => false + case _ => true + } + } + .withStats(stats.scope(name)) + .withName(name) + } + + /** + * Check if tweet contains any optouted free form interests. + * Currently, we use it for media categories and semantic core + * @param stats + * @return + */ + def noOptoutFreeFormInterestPredicate( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "free_form_interest_opt_out" + val tweetMediaAnnotationFeature = + "tweet.mediaunderstanding.tweet_annotations.safe_category_probabilities" + val tweetSemanticCoreFeature = + "tweet.core.tweet.semantic_core_annotations" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + val withOptOutFreeFormInterestsCounter = stats.counter("with_optout_interests") + val withoutOptOutInterestsCounter = stats.counter("without_optout_interests") + val withOptOutFreeFormInterestsFromMediaAnnotationCounter = + stats.counter("with_optout_interests_from_media_annotation") + val withOptOutFreeFormInterestsFromSemanticCoreCounter = + stats.counter("with_optout_interests_from_semantic_core") + Predicate + .fromAsync { candidate: PushCandidate => + val tweetSemanticCoreEntityIds = candidate.sparseBinaryFeatures + .getOrElse(tweetSemanticCoreFeature, Set.empty[String]).map { id => + id.split('.')(2) + }.toSet + val tweetMediaAnnotationIds = candidate.sparseContinuousFeatures + .getOrElse(tweetMediaAnnotationFeature, Map.empty[String, Double]).keys.toSet + + candidate.target.optOutFreeFormUserInterests.map { + case optOutUserInterests: Seq[String] => + withOptOutFreeFormInterestsCounter.incr() + val optOutUserInterestsSet = optOutUserInterests.toSet + val mediaAnnoIntersect = optOutUserInterestsSet.intersect(tweetMediaAnnotationIds) + val semanticCoreIntersect = optOutUserInterestsSet.intersect(tweetSemanticCoreEntityIds) + if (!mediaAnnoIntersect.isEmpty) { + withOptOutFreeFormInterestsFromMediaAnnotationCounter.incr() + } + if (!semanticCoreIntersect.isEmpty) { + withOptOutFreeFormInterestsFromSemanticCoreCounter.incr() + } + semanticCoreIntersect.isEmpty && mediaAnnoIntersect.isEmpty + case _ => + withoutOptOutInterestsCounter.incr() + true + } + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + def tweetCandidateWithLessThan2SocialContextsIsAReply( + implicit stats: StatsReceiver + ): NamedPredicate[TweetCandidate with TweetDetails with SocialContextActions] = { + val name = "tweet_candidate_with_less_than_2_social_contexts_is_not_a_reply" + Predicate + .from[TweetCandidate with TweetDetails with SocialContextActions] { cand => + cand.isReply match { + case Some(true) if cand.socialContextTweetIds.size < 2 => false + case _ => true + } + } + .withStats(stats.scope(name)) + .withName(name) + } + + def f1CandidateIsNotAReply(implicit stats: StatsReceiver): NamedPredicate[F1Candidate] = { + val name = "f1_candidate_is_not_a_reply" + Predicate + .from[F1Candidate] { candidate => + candidate.isReply match { + case Some(true) => false + case _ => true + } + } + .withStats(stats.scope(name)) + .withName(name) + } + + def outOfNetworkTweetCandidateEnabledCrTag( + implicit stats: StatsReceiver + ): NamedPredicate[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] = { + val name = "out_of_network_tweet_candidate_enabled_crtag" + val scopedStats = stats.scope(name) + Predicate + .from[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] { cand => + val disabledCrTag = cand.target + .params(PushFeatureSwitchParams.OONCandidatesDisabledCrTagParam) + val candGeneratedByDisabledSignal = cand.tagsCR.exists { tagsCR => + val tagsCRSet = tagsCR.map(_.toString).toSet + tagsCRSet.nonEmpty && tagsCRSet.subsetOf(disabledCrTag.toSet) + } + if (candGeneratedByDisabledSignal) { + cand.tagsCR.getOrElse(Nil).foreach(tag => scopedStats.counter(tag.toString).incr()) + false + } else true + } + .withStats(scopedStats) + .withName(name) + } + + def outOfNetworkTweetCandidateEnabledCrtGroup( + implicit stats: StatsReceiver + ): NamedPredicate[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] = { + val name = "out_of_network_tweet_candidate_enabled_crt_group" + val scopedStats = stats.scope(name) + Predicate + .from[OutOfNetworkTweetCandidate with TargetInfo[TargetUser with TargetABDecider]] { cand => + val disabledCrtGroup = cand.target + .params(PushFeatureSwitchParams.OONCandidatesDisabledCrtGroupParam) + val crtGroup = CandidateUtil.getCrtGroup(cand.commonRecType) + val candGeneratedByDisabledCrt = disabledCrtGroup.contains(crtGroup) + if (candGeneratedByDisabledCrt) { + scopedStats.counter("filter_" + crtGroup.toString).incr() + false + } else true + } + .withStats(scopedStats) + .withName(name) + } + + def outOfNetworkTweetCandidateIsNotAReply( + implicit stats: StatsReceiver + ): NamedPredicate[OutOfNetworkTweetCandidate] = { + val name = "out_of_network_tweet_candidate_is_not_a_reply" + Predicate + .from[OutOfNetworkTweetCandidate] { cand => + cand.isReply match { + case Some(true) => false + case _ => true + } + } + .withStats(stats.scope(name)) + .withName(name) + } + + def recommendedTweetIsAuthoredBySelf( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate] = + Predicate + .from[PushCandidate] { + case tweetCandidate: PushCandidate with TweetDetails => + tweetCandidate.authorId match { + case Some(authorId) => authorId != tweetCandidate.target.targetId + case None => true + } + case _ => + true + } + .withStats(statsReceiver.scope("predicate_self_author")) + .withName("self_author") + + def authorInSocialContext(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = + Predicate + .from[PushCandidate] { + case tweetCandidate: PushCandidate with TweetDetails with SocialContextActions => + tweetCandidate.authorId match { + case Some(authorId) => + !tweetCandidate.socialContextUserIds.contains(authorId) + case None => true + } + case _ => true + } + .withStats(statsReceiver.scope("predicate_author_social_context")) + .withName("author_social_context") + + def selfInSocialContext(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { + val name = "self_social_context" + Predicate + .from[PushCandidate] { + case candidate: PushCandidate with SocialContextActions => + !candidate.socialContextUserIds.contains(candidate.target.targetId) + case _ => + true + } + .withStats(statsReceiver.scope(s"${name}_predicate")) + .withName(name) + } + + def minSocialContext( + threshold: Int + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with SocialContextActions] = { + Predicate + .from { candidate: PushCandidate with SocialContextActions => + candidate.socialContextUserIds.size >= threshold + } + .withStats(statsReceiver.scope("predicate_min_social_context")) + .withName("min_social_context") + } + + private def anyWithheldContent( + userStore: ReadableStore[Long, User], + userCountryStore: ReadableStore[Long, Location] + )( + implicit statsReceiver: StatsReceiver + ): Predicate[TargetRecUser] = + GizmoduckUserPredicate.withheldContentPredicate( + userStore = userStore, + userCountryStore = userCountryStore, + statsReceiver = statsReceiver, + checkAllCountries = true + ) + + def targetUserExists(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { + TargetUserPredicates + .targetUserExists()(statsReceiver) + .flatContraMap { candidate: PushCandidate => Future.value(candidate.target) } + .withName("target_user_exists") + } + + def secondaryDormantAccountPredicate( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "secondary_dormant_account" + TargetUserPredicates + .secondaryDormantAccountPredicate()(statsReceiver) + .on { candidate: PushCandidate => candidate.target } + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } + + def socialContextBeingFollowed( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with SocialContextActions] = + SocialGraphPredicate + .allRelationEdgesExist(edgeStore, RelationshipType.Following) + .on { candidate: PushCandidate with SocialContextActions => + candidate.socialContextUserIds.map { u => Edge(candidate.target.targetId, u) } + } + .withStats(statsReceiver.scope("predicate_social_context_being_followed")) + .withName("social_context_being_followed") + + private def edgeFromCandidate(candidate: PushCandidate with TweetAuthor): Option[Edge] = { + candidate.authorId map { authorId => Edge(candidate.target.targetId, authorId) } + } + + def authorNotBeingDeviceFollowed( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor] = { + SocialGraphPredicate + .relationExists(edgeStore, RelationshipType.DeviceFollowing) + .optionalOn( + edgeFromCandidate, + missingResult = false + ) + .flip + .withStats(statsReceiver.scope("predicate_author_not_device_followed")) + .withName("author_not_device_followed") + } + + def authorBeingFollowed( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor] = { + SocialGraphPredicate + .relationExists(edgeStore, RelationshipType.Following) + .optionalOn( + edgeFromCandidate, + missingResult = false + ) + .withStats(statsReceiver.scope("predicate_author_being_followed")) + .withName("author_being_followed") + } + + def authorNotBeingFollowed( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor] = { + SocialGraphPredicate + .relationExists(edgeStore, RelationshipType.Following) + .optionalOn( + edgeFromCandidate, + missingResult = false + ) + .flip + .withStats(statsReceiver.scope("predicate_author_not_being_followed")) + .withName("author_not_being_followed") + } + + def recommendedTweetAuthorAcceptableToTargetUser( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor] = { + val name = "recommended_tweet_author_acceptable_to_target_user" + SocialGraphPredicate + .anyRelationExists( + edgeStore, + Set( + RelationshipType.Blocking, + RelationshipType.BlockedBy, + RelationshipType.HideRecommendations, + RelationshipType.Muting + ) + ) + .flip + .optionalOn( + edgeFromCandidate, + missingResult = false + ) + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } + + def relationNotExistsPredicate( + edgeStore: ReadableStore[RelationEdge, Boolean], + relations: Set[RelationshipType] + ): Predicate[(Long, Iterable[Long])] = + SocialGraphPredicate + .anyRelationExistsForMultiEdge( + edgeStore, + relations + ) + .flip + .on { + case (targetUserId, userIds) => + MultiEdge(targetUserId, userIds.toSet) + } + + def blocking(edgeStore: ReadableStore[RelationEdge, Boolean]): Predicate[(Long, Iterable[Long])] = + relationNotExistsPredicate( + edgeStore, + Set(RelationshipType.BlockedBy, RelationshipType.Blocking) + ) + + def blockingOrMuting( + edgeStore: ReadableStore[RelationEdge, Boolean] + ): Predicate[(Long, Iterable[Long])] = + relationNotExistsPredicate( + edgeStore, + Set(RelationshipType.BlockedBy, RelationshipType.Blocking, RelationshipType.Muting) + ) + + def socialContextNotRetweetFollowing( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with SocialContextActions] = { + val name = "social_context_not_retweet_following" + relationNotExistsPredicate(edgeStore, Set(RelationshipType.NotRetweetFollowing)) + .optionalOn[PushCandidate with SocialContextActions]( + { + case candidate: PushCandidate with SocialContextActions + if RecTypes.isTweetRetweetType(candidate.commonRecType) => + Some((candidate.target.targetId, candidate.socialContextUserIds)) + case _ => + None + }, + missingResult = true + ) + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } + + def socialContextBlockingOrMuting( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with SocialContextActions] = + blockingOrMuting(edgeStore) + .on { candidate: PushCandidate with SocialContextActions => + (candidate.target.targetId, candidate.socialContextUserIds) + } + .withStats(statsReceiver.scope("predicate_social_context_blocking_or_muting")) + .withName("social_context_blocking_or_muting") + + /** + * Use hyrated Tweet object for F1 Protected experiment for checking null cast as Tweetypie hydration + * fails for protected Authors without passing in Target id. We do this specifically for + * F1 Protected Tweet Experiment in Earlybird Adaptor. + * For rest of the traffic refer to existing Nullcast Predicate + */ + def nullCastF1ProtectedExperientPredicate( + tweetypieStore: ReadableStore[Long, TweetyPieResult] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate with TweetDetails] = { + val name = "f1_exempted_null_cast_tweet" + val f1NullCastCheckCounter = statsReceiver.scope(name).counter("f1_null_cast_check") + Predicate + .fromAsync { tweetCandidate: PushCandidate with TweetCandidate with TweetDetails => + if (RecTypes.f1FirstDegreeTypes(tweetCandidate.commonRecType) && tweetCandidate.target + .params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors)) { + f1NullCastCheckCounter.incr() + tweetCandidate.tweet match { + case Some(tweetObj) => + baseNullCastTweet().apply(Seq(TweetyPieResult(tweetObj, None, None))).map(_.head) + case _ => Future.False + } + } else { + nullCastTweet(tweetypieStore).apply(Seq(tweetCandidate)).map(_.head) + } + } + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } + + private def baseNullCastTweet(): Predicate[TweetyPieResult] = + Predicate.from { t: TweetyPieResult => !t.tweet.coreData.exists { cd => cd.nullcast } } + + def nullCastTweet( + tweetyPieStore: ReadableStore[Long, TweetyPieResult] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate] = { + val name = "null_cast_tweet" + baseNullCastTweet() + .flatOptionContraMap[PushCandidate with TweetCandidate]( + f = (tweetCandidate: PushCandidate + with TweetCandidate) => tweetyPieStore.get(tweetCandidate.tweetId), + missingResult = false + ) + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } + + /** + * Use the predicate except fn is true. + */ + def exceptedPredicate[T <: PushCandidate]( + name: String, + fn: T => Future[Boolean], + predicate: Predicate[T] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + Predicate + .fromAsync { e: T => fn(e) } + .or(predicate) + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + /** + * + * @param edgeStore [[ReadableStore[RelationEdge, Boolean]]] + * @return - allow only out-network tweets if in-network tweets are disabled + */ + def disableInNetworkTweetPredicate( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor] = { + val name = "disable_in_network_tweet" + Predicate + .fromAsync { candidate: PushCandidate with TweetAuthor => + if (candidate.target.params(PushParams.DisableInNetworkTweetCandidatesParam)) { + authorNotBeingFollowed(edgeStore) + .apply(Seq(candidate)) + .map(_.head) + } else Future.True + }.withStats(statsReceiver.scope(name)) + .withName(name) + } + + /** + * + * @param edgeStore [[ReadableStore[RelationEdge, Boolean]]] + * @return - allow only in-network tweets if out-network tweets are disabled + */ + def disableOutNetworkTweetPredicate( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor] = { + val name = "disable_out_network_tweet" + Predicate + .fromAsync { candidate: PushCandidate with TweetAuthor => + if (candidate.target.params(PushFeatureSwitchParams.DisableOutNetworkTweetCandidatesFS)) { + authorBeingFollowed(edgeStore) + .apply(Seq(candidate)) + .map(_.head) + } else Future.True + }.withStats(statsReceiver.scope(name)) + .withName(name) + } + + def alwaysTruePredicate: NamedPredicate[PushCandidate] = { + Predicate + .all[PushCandidate] + .withName("predicate_AlwaysTrue") + } + + def alwaysTruePushCandidatePredicate: NamedPredicate[PushCandidate] = { + Predicate + .all[PushCandidate] + .withName("predicate_AlwaysTrue") + } + + def alwaysFalsePredicate(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { + val name = "predicate_AlwaysFalse" + val scopedStatsReceiver = statsReceiver.scope(name) + Predicate + .from { candidate: PushCandidate => false } + .withStats(scopedStatsReceiver) + .withName(name) + } + + def accountCountryPredicate( + allowedCountries: Set[String] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "AccountCountryPredicate" + val stats = statsReceiver.scope(name) + AccountCountryPredicate(allowedCountries) + .on { candidate: PushCandidate => candidate.target } + .withStats(stats) + .withName(name) + } + + def paramPredicate[T <: PushCandidate]( + param: Param[Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + val name = param.getClass.getSimpleName.stripSuffix("$") + TargetPredicates + .paramPredicate(param) + .on { candidate: PushCandidate => candidate.target } + .withStats(statsReceiver.scope(s"param_${name}_controlled_predicate")) + .withName(s"param_${name}_controlled_predicate") + } + + def isDeviceEligibleForNewsOrSports( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "is_device_eligible_for_news_or_sports" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + Predicate + .fromAsync { candidate: PushCandidate => + candidate.target.deviceInfo.map(_.exists(_.isNewsEligible)) + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + def isDeviceEligibleForCreatorPush( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "is_device_eligible_for_creator_push" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + Predicate + .fromAsync { candidate: PushCandidate => + candidate.target.deviceInfo.map(_.exists(settings => + settings.isNewsEligible || settings.isRecommendationsEligible)) + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + /** + * Like [[TargetUserPredicates.homeTimelineFatigue()]] but for candidate. + */ + def htlFatiguePredicate( + fatigueDuration: Param[Duration] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "htl_fatigue" + Predicate + .fromAsync { candidate: PushCandidate => + val _fatigueDuration = candidate.target.params(fatigueDuration) + TargetUserPredicates + .homeTimelineFatigue( + fatigueDuration = _fatigueDuration + ).apply(Seq(candidate.target)).map(_.head) + } + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + def mrWebHoldbackPredicate( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "mr_web_holdback_for_candidate" + val scopedStats = stats.scope(name) + PredicatesForCandidate.exludeCrtFromPushHoldback + .or( + TargetPredicates + .webNotifsHoldback() + .on { candidate: PushCandidate => candidate.target } + ) + .withStats(scopedStats) + .withName(name) + } + + def candidateEnabledForEmailPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "candidates_enabled_for_email" + Predicate + .from { candidate: PushCandidate => + if (candidate.target.isEmailUser) + candidate.isInstanceOf[TweetCandidate with TweetAuthor with RecommendationType] + else true + } + .withStats(stats.scope(name)) + .withName(name) + } + + def protectedTweetF1ExemptPredicate[ + T <: TargetUser with TargetABDecider, + Cand <: TweetCandidate with TweetAuthorDetails with TargetInfo[T] + ]( + implicit stats: StatsReceiver + ): NamedPredicate[ + TweetCandidate with TweetAuthorDetails with TargetInfo[ + TargetUser with TargetABDecider + ] + ] = { + val name = "f1_exempt_tweet_author_protected" + val skipForProtectedAuthorScope = stats.scope(name).scope("skip_protected_author_for_f1") + val authorIsProtectedCounter = skipForProtectedAuthorScope.counter("author_protected_true") + val authorIsNotProtectedCounter = skipForProtectedAuthorScope.counter("author_protected_false") + val authorNotFoundCounter = stats.scope(name).counter("author_not_found") + Predicate + .fromAsync[TweetCandidate with TweetAuthorDetails with TargetInfo[ + TargetUser with TargetABDecider + ]] { + case candidate: F1Candidate + if candidate.target.params(PushFeatureSwitchParams.EnableF1FromProtectedTweetAuthors) => + candidate.tweetAuthor.foreach { + case Some(author) => + if (GizmoduckUserPredicate.isProtected(author)) { + authorIsProtectedCounter.incr() + } else authorIsNotProtectedCounter.incr() + case _ => authorNotFoundCounter.incr() + } + Future.True + case cand => + TweetAuthorPredicates.recTweetAuthorProtected.apply(Seq(cand)).map(_.head) + } + .withStats(stats.scope(name)) + .withName(name) + } + + /** + * filter a notification if user has already received ANY prior notification about the space id + * @param stats + * @return + */ + def duplicateSpacesPredicate( + implicit stats: StatsReceiver + ): NamedPredicate[Space with PushCandidate] = { + val name = "duplicate_spaces_predicate" + Predicate + .fromAsync { c: Space with PushCandidate => + c.target.pushRecItems.map { pushRecItems => + !pushRecItems.spaceIds.contains(c.spaceId) + } + } + .withStats(stats.scope(name)) + .withName(name) + } + + def filterOONCandidatePredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "filter_oon_candidate" + + Predicate + .fromAsync[PushCandidate] { cand => + val crt = cand.commonRecType + val isOONCandidate = + RecTypes.isOutOfNetworkTweetRecType(crt) || RecTypes.outOfNetworkTopicTweetTypes + .contains(crt) || RecTypes.isOutOfNetworkSpaceType(crt) || RecTypes.userTypes.contains( + crt) + if (isOONCandidate) { + cand.target.notificationsFromOnlyPeopleIFollow.map { inNetworkOnly => + if (inNetworkOnly) { + stats.scope(name, crt.toString).counter("inNetworkOnlyOn").incr() + } else { + stats.scope(name, crt.toString).counter("inNetworkOnlyOff").incr() + } + !(inNetworkOnly && cand.target.params( + PushFeatureSwitchParams.EnableOONFilteringBasedOnUserSettings)) + } + } else Future.True + } + .withStats(stats.scope(name)) + .withName(name) + } + + def exludeCrtFromPushHoldback( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = Predicate + .from { candidate: PushCandidate => + val crtName = candidate.commonRecType.name + val target = candidate.target + target + .params(PushFeatureSwitchParams.CommonRecommendationTypeDenyListPushHoldbacks) + .exists(crtName.equalsIgnoreCase) + } + .withStats(stats.scope("exclude_crt_from_push_holdbacks")) + + def enableSendHandlerCandidates(implicit stats: StatsReceiver): NamedPredicate[PushCandidate] = { + val name = "sendhandler_enable_push_recommendations" + PredicatesForCandidate.exludeCrtFromPushHoldback + .or(PredicatesForCandidate.paramPredicate( + PushFeatureSwitchParams.EnablePushRecommendationsParam)) + .withStats(stats.scope(name)) + .withName(name) + } + + def openAppExperimentUserCandidateAllowList( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "open_app_experiment_user_candidate_allow_list" + Predicate + .fromAsync { candidate: PushCandidate => + val target = candidate.target + Future.join(target.isOpenAppExperimentUser, target.targetUser).map { + case (isOpenAppUser, targetUser) => + val shouldLimitOpenAppCrts = + isOpenAppUser || targetUser.exists(_.userType == UserType.Soft) + + if (shouldLimitOpenAppCrts) { + val listOfAllowedCrt = target + .params(PushFeatureSwitchParams.ListOfCrtsForOpenApp) + .flatMap(CommonRecommendationType.valueOf) + listOfAllowedCrt.contains(candidate.commonRecType) + } else true + } + }.withStats(stats.scope(name)) + .withName(name) + } + + def isTargetBlueVerified( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "is_target_already_blue_verified" + Predicate + .fromAsync { candidate: PushCandidate => + val target = candidate.target + target.isBlueVerified.map(_.getOrElse(false)) + }.withStats(stats.scope(name)) + .withName(name) + } + + def isTargetLegacyVerified( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "is_target_already_legacy_verified" + Predicate + .fromAsync { candidate: PushCandidate => + val target = candidate.target + target.isVerified.map(_.getOrElse(false)) + }.withStats(stats.scope(name)) + .withName(name) + } + + def isTargetSuperFollowCreator(implicit stats: StatsReceiver): NamedPredicate[PushCandidate] = { + val name = "is_target_already_super_follow_creator" + Predicate + .fromAsync { candidate: PushCandidate => + val target = candidate.target + target.isSuperFollowCreator.map( + _.getOrElse(false) + ) + }.withStats(stats.scope(name)) + .withName(name) + } + + def isChannelValidPredicate( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "is_channel_valid" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + Predicate + .fromAsync { candidate: PushCandidate => + candidate + .getChannels().map(channels => + !(channels.toSet.size == 1 && channels.head == ChannelName.None)) + } + .withStats(scopedStatsReceiver) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.scala new file mode 100644 index 0000000000..e335c8d9c3 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SGSPredicatesForCandidate.scala @@ -0,0 +1,174 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.SocialGraphServiceRelationshipMap +import com.twitter.frigate.common.base.TweetAuthor +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.gizmoduck.thriftscala.UserType +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.hermit.predicate.socialgraph.Edge +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.socialgraph.thriftscala.RelationshipType +import com.twitter.util.Future + +/** + * Refactor SGS predicates so that predicates can use relationshipMap we generate in hydrate step + */ +object SGSPredicatesForCandidate { + + case class RelationshipMapEdge(edge: Edge, relationshipMap: Map[RelationEdge, Boolean]) + + private def relationshipMapEdgeFromCandidate( + candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap + ): Option[RelationshipMapEdge] = { + candidate.authorId map { authorId => + RelationshipMapEdge(Edge(candidate.target.targetId, authorId), candidate.relationshipMap) + } + } + + def authorBeingFollowed( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { + val name = "author_not_being_followed" + val stats = statsReceiver.scope(name) + val softUserCounter = stats.counter("soft_user") + + val sgsAuthorBeingFollowedPredicate = Predicate + .from { relationshipMapEdge: RelationshipMapEdge => + anyRelationExist(relationshipMapEdge, Set(RelationshipType.Following)) + } + + Predicate + .fromAsync { + candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap => + val target = candidate.target + target.targetUser.flatMap { + case Some(gizmoduckUser) if gizmoduckUser.userType == UserType.Soft => + softUserCounter.incr() + target.seedsWithWeight.map { followedUsersWithWeightOpt => + candidate.authorId match { + case Some(authorId) => + val followedUsers = followedUsersWithWeightOpt.getOrElse(Map.empty).keys + followedUsers.toSet.contains(authorId) + + case None => false + } + } + + case _ => + sgsAuthorBeingFollowedPredicate + .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) + .apply(Seq(candidate)) + .map(_.head) + } + }.withStats(stats) + .withName(name) + } + + def authorNotBeingDeviceFollowed( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { + val name = "author_being_device_followed" + Predicate + .from { relationshipMapEdge: RelationshipMapEdge => + { + anyRelationExist(relationshipMapEdge, Set(RelationshipType.DeviceFollowing)) + } + } + .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) + .flip + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + def recommendedTweetAuthorAcceptableToTargetUser( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { + val name = "recommended_tweet_author_not_acceptable_to_target_user" + Predicate + .from { relationshipMapEdge: RelationshipMapEdge => + { + anyRelationExist( + relationshipMapEdge, + Set( + RelationshipType.Blocking, + RelationshipType.BlockedBy, + RelationshipType.HideRecommendations, + RelationshipType.Muting + )) + } + } + .flip + .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + def authorNotBeingFollowed( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { + Predicate + .from { relationshipMapEdge: RelationshipMapEdge => + { + anyRelationExist(relationshipMapEdge, Set(RelationshipType.Following)) + } + } + .optionalOn(relationshipMapEdgeFromCandidate, missingResult = false) + .flip + .withStats(statsReceiver.scope("predicate_author_not_being_followed_pre_ranking")) + .withName("author_not_being_followed") + } + + def disableInNetworkTweetPredicate( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { + val name = "enable_in_network_tweet" + Predicate + .fromAsync { + candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap => + if (candidate.target.params(PushParams.DisableInNetworkTweetCandidatesParam)) { + authorNotBeingFollowed + .apply(Seq(candidate)) + .map(_.head) + } else Future.True + }.withStats(statsReceiver.scope(name)) + .withName(name) + } + + def disableOutNetworkTweetPredicate( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap] = { + val name = "enable_out_network_tweet" + Predicate + .fromAsync { + candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap => + if (candidate.target.params(PushFeatureSwitchParams.DisableOutNetworkTweetCandidatesFS)) { + authorBeingFollowed + .apply(Seq(candidate)) + .map(_.head) + } else Future.True + }.withStats(statsReceiver.scope(name)) + .withName(name) + } + + /** + * Returns true if the provided relationshipEdge exists among + * @param candidate candidate + * @param relationships relaionships + * @return Boolean result + */ + private def anyRelationExist( + relationshipMapEdge: RelationshipMapEdge, + relationships: Set[RelationshipType] + ): Boolean = { + val resultSeq = relationships.map { relationship => + relationshipMapEdge.relationshipMap.getOrElse( + RelationEdge(relationshipMapEdge.edge, relationship), + false) + }.toSeq + resultSeq.contains(true) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.scala new file mode 100644 index 0000000000..a4728eba29 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ScarecrowPredicate.scala @@ -0,0 +1,138 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.thriftscala._ +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.scarecrow.{ScarecrowPredicate => HermitScarecrowPredicate} +import com.twitter.relevance.feature_store.thriftscala.FeatureData +import com.twitter.relevance.feature_store.thriftscala.FeatureValue +import com.twitter.service.gen.scarecrow.thriftscala.Event +import com.twitter.service.gen.scarecrow.thriftscala.TieredActionResult +import com.twitter.storehaus.ReadableStore + +object ScarecrowPredicate { + val name = "" + + def candidateToEvent(candidate: PushCandidate): Event = { + val recommendedUserIdOpt = candidate match { + case tweetCandidate: TweetCandidate with TweetAuthor => + tweetCandidate.authorId + case userCandidate: UserCandidate => + Some(userCandidate.userId) + case _ => None + } + val hashtagsInTweet = candidate match { + case tweetCandidate: TweetCandidate with TweetDetails => + tweetCandidate.tweetyPieResult + .flatMap { tweetPieResult => + tweetPieResult.tweet.hashtags.map(_.map(_.text)) + }.getOrElse(Nil) + case _ => + Nil + } + val urlsInTweet = candidate match { + case tweetCandidate: TweetCandidate with TweetDetails => + tweetCandidate.tweetyPieResult + .flatMap { tweetPieResult => + tweetPieResult.tweet.urls.map(_.flatMap(_.expanded)) + } + case _ => None + } + val tweetIdOpt = candidate match { + case tweetCandidate: TweetCandidate => + Some(tweetCandidate.tweetId) + case _ => + None + } + val urlOpt = candidate match { + case candidate: UrlCandidate => + Some(candidate.url) + case _ => + None + } + val scUserIds = candidate match { + case hasSocialContext: SocialContextActions => Some(hasSocialContext.socialContextUserIds) + case _ => None + } + + val eventTitleOpt = candidate match { + case eventCandidate: EventCandidate with EventDetails => + Some(eventCandidate.eventTitle) + case _ => + None + } + + val urlTitleOpt = candidate match { + case candidate: UrlCandidate => + candidate.title + case _ => + None + } + + val urlDescriptionOpt = candidate match { + case candidate: UrlCandidate with UrlCandidateWithDetails => + candidate.description + case _ => + None + } + + Event( + "magicrecs_recommendation_write", + Map( + "targetUserId" -> FeatureData(Some(FeatureValue.LongValue(candidate.target.targetId))), + "type" -> FeatureData( + Some( + FeatureValue.StrValue(candidate.commonRecType.name) + ) + ), + "recommendedUserId" -> FeatureData(recommendedUserIdOpt map { id => + FeatureValue.LongValue(id) + }), + "tweetId" -> FeatureData(tweetIdOpt map { id => + FeatureValue.LongValue(id) + }), + "url" -> FeatureData(urlOpt map { url => + FeatureValue.StrValue(url) + }), + "hashtagsInTweet" -> FeatureData(Some(FeatureValue.StrListValue(hashtagsInTweet))), + "urlsInTweet" -> FeatureData(urlsInTweet.map(FeatureValue.StrListValue)), + "socialContexts" -> FeatureData(scUserIds.map { sc => + FeatureValue.LongListValue(sc) + }), + "eventTitle" -> FeatureData(eventTitleOpt.map { eventTitle => + FeatureValue.StrValue(eventTitle) + }), + "urlTitle" -> FeatureData(urlTitleOpt map { title => + FeatureValue.StrValue(title) + }), + "urlDescription" -> FeatureData(urlDescriptionOpt map { des => + FeatureValue.StrValue(des) + }) + ) + ) + } + + def candidateToPossibleEvent(c: PushCandidate): Option[Event] = { + if (c.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) { + Some(candidateToEvent(c)) + } else { + None + } + } + + def apply( + scarecrowCheckEventStore: ReadableStore[Event, TieredActionResult] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate] = { + HermitScarecrowPredicate(scarecrowCheckEventStore) + .optionalOn( + candidateToPossibleEvent, + missingResult = true + ) + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.scala new file mode 100644 index 0000000000..044c0afdb6 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/SpacePredicate.scala @@ -0,0 +1,153 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.SpaceCandidate +import com.twitter.frigate.common.base.SpaceCandidateDetails +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.hermit.predicate.socialgraph.Edge +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.hermit.predicate.socialgraph.SocialGraphPredicate +import com.twitter.socialgraph.thriftscala.RelationshipType +import com.twitter.storehaus.ReadableStore +import com.twitter.strato.response.Err +import com.twitter.ubs.thriftscala.AudioSpace +import com.twitter.ubs.thriftscala.BroadcastState +import com.twitter.ubs.thriftscala.ParticipantUser +import com.twitter.ubs.thriftscala.Participants +import com.twitter.util.Future + +object SpacePredicate { + + /** Filters the request if the target is present in the space as a listener, speakeTestConfigr, or admin */ + def targetInSpace( + audioSpaceParticipantsStore: ReadableStore[String, Participants] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[SpaceCandidateDetails with RawCandidate] = { + val name = "target_in_space" + Predicate + .fromAsync[SpaceCandidateDetails with RawCandidate] { spaceCandidate => + audioSpaceParticipantsStore.get(spaceCandidate.spaceId).map { + case Some(participants) => + val allParticipants: Seq[ParticipantUser] = + (participants.admins ++ participants.speakers ++ participants.listeners).flatten.toSeq + val isInSpace = allParticipants.exists { participant => + participant.twitterUserId.contains(spaceCandidate.target.targetId) + } + !isInSpace + case None => false + } + }.withStats(statsReceiver.scope(name)) + .withName(name) + } + + /** + * + * @param audioSpaceStore: space metadata store + * @param statsReceiver: record stats + * @return: true if the space not started ELSE false to filter out notification + */ + def scheduledSpaceStarted( + audioSpaceStore: ReadableStore[String, AudioSpace] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[SpaceCandidate with RawCandidate] = { + val name = "scheduled_space_started" + Predicate + .fromAsync[SpaceCandidate with RawCandidate] { spaceCandidate => + audioSpaceStore + .get(spaceCandidate.spaceId) + .map(_.exists(_.state.contains(BroadcastState.NotStarted))) + .rescue { + case Err(Err.Authorization, _, _) => + Future.False + } + } + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + private def relationshipMapEdgeFromSpaceCandidate( + candidate: RawCandidate with SpaceCandidate + ): Option[(Long, Seq[Long])] = { + candidate.hostId.map { spaceHostId => + (candidate.target.targetId, Seq(spaceHostId)) + } + } + + /** + * Check only host block for scheduled space reminders + * @return: True if no blocking relation between host and target user, else False + */ + def spaceHostTargetUserBlocking( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[SpaceCandidate with RawCandidate] = { + val name = "space_host_target_user_blocking" + PredicatesForCandidate + .blocking(edgeStore) + .optionalOn(relationshipMapEdgeFromSpaceCandidate, false) + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + private def edgeFromCandidate( + candidate: PushCandidate with TweetAuthorDetails + ): Future[Option[Edge]] = { + candidate.tweetAuthor.map(_.map { author => Edge(candidate.target.targetId, author.id) }) + } + + def recommendedTweetAuthorAcceptableToTargetUser( + edgeStore: ReadableStore[RelationEdge, Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetAuthorDetails] = { + val name = "recommended_tweet_author_acceptable_to_target_user" + SocialGraphPredicate + .anyRelationExists( + edgeStore, + Set( + RelationshipType.Blocking, + RelationshipType.BlockedBy, + RelationshipType.HideRecommendations, + RelationshipType.Muting + ) + ) + .flip + .flatOptionContraMap( + edgeFromCandidate, + missingResult = false + ) + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } + + def narrowCastSpace( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[SpaceCandidateDetails with RawCandidate] = { + val name = "narrow_cast_space" + val narrowCastSpaceScope = statsReceiver.scope(name) + val employeeSpaceCounter = narrowCastSpaceScope.counter("employees") + val superFollowerSpaceCounter = narrowCastSpaceScope.counter("super_followers") + + Predicate + .fromAsync[SpaceCandidateDetails with RawCandidate] { candidate => + candidate.audioSpaceFut.map { + case Some(audioSpace) if audioSpace.narrowCastSpaceType.contains(1L) => + employeeSpaceCounter.incr() + candidate.target.params(PushFeatureSwitchParams.EnableEmployeeOnlySpaceNotifications) + case Some(audioSpace) if audioSpace.narrowCastSpaceType.contains(2L) => + superFollowerSpaceCounter.incr() + false + case _ => true + } + }.withStats(narrowCastSpaceScope) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.scala new file mode 100644 index 0000000000..d02e5b89ad --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetEngagementPredicate.scala @@ -0,0 +1,27 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.tweetypie.EngagementsPredicate +import com.twitter.hermit.predicate.tweetypie.Perspective +import com.twitter.hermit.predicate.tweetypie.UserTweet +import com.twitter.storehaus.ReadableStore + +object TargetEngagementPredicate { + val name = "target_engagement" + def apply( + perspectiveStore: ReadableStore[UserTweet, Perspective], + defaultForMissing: Boolean + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate] = { + EngagementsPredicate(perspectiveStore, defaultForMissing) + .on { candidate: PushCandidate with TweetCandidate => + UserTweet(candidate.target.targetId, candidate.tweetId) + } + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.scala new file mode 100644 index 0000000000..c5042bbc8d --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetNtabCaretClickFatiguePredicate.scala @@ -0,0 +1,91 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TargetUser +import com.twitter.frigate.common.candidate.CaretFeedbackHistory +import com.twitter.frigate.common.candidate.FrigateHistory +import com.twitter.frigate.common.candidate.HTLVisitHistory +import com.twitter.frigate.common.candidate.TargetABDecider +import com.twitter.frigate.common.history.History +import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate.TimeSeries +import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.common.util.FeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.frigate.common.predicate.{FatiguePredicate => CommonFatiguePredicate} + +object TargetNtabCaretClickFatiguePredicate { + import NtabCaretClickFatiguePredicateHelper._ + + private val MagicRecsCategory = "MagicRecs" + + def apply[ + T <: TargetUser with TargetABDecider with CaretFeedbackHistory with FrigateHistory with HTLVisitHistory + ]( + filterHistory: TimeSeries => TimeSeries = + CommonFatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), + filterCaretFeedbackHistory: TargetUser with TargetABDecider with CaretFeedbackHistory => Seq[ + CaretFeedbackDetails + ] => Seq[CaretFeedbackDetails] = + CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), + calculateFatiguePeriod: Seq[CaretFeedbackDetails] => Duration = calculateFatiguePeriodMagicRecs, + useMostRecentDislikeTime: Boolean = false, + name: String = "NtabCaretClickFatiguePredicate" + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + + val scopedStats = statsReceiver.scope(name) + val crtStats = scopedStats.scope("crt") + Predicate + .fromAsync { target: T => + Future.join(target.history, target.caretFeedbacks).map { + case (history, Some(feedbackDetails)) => { + val feedbackDetailsDeduped = dedupFeedbackDetails( + filterCaretFeedbackHistory(target)(feedbackDetails), + scopedStats + ) + + val fatiguePeriod = + if (hasUserDislikeInLast30Days(feedbackDetailsDeduped) && target.params( + PushFeatureSwitchParams.EnableReducedFatigueRulesForSeeLessOften)) { + durationToFilterMRForSeeLessOftenExpt( + feedbackDetailsDeduped, + target.params(FeatureSwitchParams.NumberOfDaysToFilterMRForSeeLessOften), + target.params(FeatureSwitchParams.NumberOfDaysToReducePushCapForSeeLessOften), + scopedStats + ) + } else { + calculateFatiguePeriod(feedbackDetailsDeduped) + } + + val crtlist = feedbackDetailsDeduped + .flatMap { fd => + fd.genericNotificationMetadata.map { gm => + gm.genericType.name + } + }.distinct.sorted.mkString("-") + + if (fatiguePeriod > 0.days) { + crtStats.scope(crtlist).counter("fatigued").incr() + } else { + crtStats.scope(crtlist).counter("non_fatigued").incr() + } + + val hasRecentSent = + hasRecentSend(History(filterHistory(history.history.toSeq).toMap), fatiguePeriod) + !hasRecentSent + } + case _ => true + } + } + .withStats(scopedStats) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.scala new file mode 100644 index 0000000000..45d0b75787 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TargetPredicates.scala @@ -0,0 +1,292 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TargetUser +import com.twitter.frigate.common.candidate.FrigateHistory +import com.twitter.frigate.common.candidate.HTLVisitHistory +import com.twitter.frigate.common.candidate.TargetABDecider +import com.twitter.frigate.common.candidate.UserDetails +import com.twitter.frigate.common.predicate.TargetUserPredicates +import com.twitter.frigate.common.predicate.{FatiguePredicate => CommonFatiguePredicate} +import com.twitter.frigate.common.store.deviceinfo.MobileClientType +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.target.TargetScoringDetails +import com.twitter.frigate.pushservice.util.PushCapUtil +import com.twitter.frigate.thriftscala.NotificationDisplayLocation +import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.timelines.configapi.FSBoundedParam +import com.twitter.timelines.configapi.Param +import com.twitter.util.Duration +import com.twitter.util.Future + +object TargetPredicates { + + def paramPredicate[T <: Target]( + param: Param[Boolean] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + val name = param.getClass.getSimpleName.stripSuffix("$") + Predicate + .from { target: T => target.params(param) } + .withStats(statsReceiver.scope(s"param_${name}_controlled_predicate")) + .withName(s"param_${name}_controlled_predicate") + } + + /** + * Use the predicate except fn is true., Same as the candidate version but for Target + */ + def exceptedPredicate[T <: TargetUser]( + name: String, + fn: T => Future[Boolean], + predicate: Predicate[T] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + Predicate + .fromAsync { e: T => fn(e) } + .or(predicate) + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + /** + * Refresh For push handler target user predicate to fatigue on visiting Home timeline + */ + def targetHTLVisitPredicate[ + T <: TargetUser with UserDetails with TargetABDecider with HTLVisitHistory + ]( + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + val name = "target_htl_visit_predicate" + Predicate + .fromAsync { target: T => + val hoursToFatigue = target.params(PushFeatureSwitchParams.HTLVisitFatigueTime) + TargetUserPredicates + .homeTimelineFatigue(hoursToFatigue.hours) + .apply(Seq(target)) + .map(_.head) + } + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + def targetPushBitEnabledPredicate[T <: Target]( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + val name = "push_bit_enabled" + val scopedStats = statsReceiver.scope(s"targetpredicate_$name") + + Predicate + .fromAsync { target: T => + target.deviceInfo + .map { info => + info.exists { deviceInfo => + deviceInfo.isRecommendationsEligible || + deviceInfo.isNewsEligible || + deviceInfo.isTopicsEligible || + deviceInfo.isSpacesEligible + } + } + }.withStats(scopedStats) + .withName(name) + } + + def targetFatiguePredicate[T <: Target]( + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + val name = "target_fatigue_predicate" + val predicateStatScope = statsReceiver.scope(name) + Predicate + .fromAsync { target: T => + PushCapUtil + .getPushCapFatigue(target, predicateStatScope) + .flatMap { pushCapInfo => + CommonFatiguePredicate + .magicRecsPushTargetFatiguePredicate( + interval = pushCapInfo.fatigueInterval, + maxInInterval = pushCapInfo.pushcap + ) + .apply(Seq(target)) + .map(_.headOption.getOrElse(false)) + } + } + .withStats(predicateStatScope) + .withName(name) + } + + def teamExceptedPredicate[T <: TargetUser]( + predicate: NamedPredicate[T] + )( + implicit stats: StatsReceiver + ): NamedPredicate[T] = { + Predicate + .fromAsync { t: T => t.isTeamMember } + .or(predicate) + .withStats(stats.scope(predicate.name)) + .withName(predicate.name) + } + + def targetValidMobileSDKPredicate[T <: Target]( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + val name = "valid_mobile_sdk" + val scopedStats = statsReceiver.scope(s"targetpredicate_$name") + + Predicate + .fromAsync { target: T => + TargetUserPredicates.validMobileSDKPredicate + .apply(Seq(target)).map(_.headOption.getOrElse(false)) + }.withStats(scopedStats) + .withName(name) + } + + def magicRecsMinDurationSinceSent[T <: Target]( + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + val name = "target_min_duration_since_push" + Predicate + .fromAsync { target: T => + PushCapUtil.getMinDurationSincePush(target, statsReceiver).flatMap { minDurationSincePush => + CommonFatiguePredicate + .magicRecsMinDurationSincePush(interval = minDurationSincePush) + .apply(Seq(target)).map(_.head) + } + } + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + def optoutProbPredicate[ + T <: TargetUser with TargetABDecider with TargetScoringDetails with FrigateHistory + ]( + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[T] = { + val name = "target_has_high_optout_probability" + Predicate + .fromAsync { target: T => + val isNewUser = target.is30DayNewUserFromSnowflakeIdTime + if (isNewUser) { + statsReceiver.scope(name).counter("all_new_users").incr() + } + target.bucketOptoutProbability + .flatMap { + case Some(optoutProb) => + if (optoutProb >= target.params(PushFeatureSwitchParams.BucketOptoutThresholdParam)) { + CommonFatiguePredicate + .magicRecsPushTargetFatiguePredicate( + interval = 24.hours, + maxInInterval = target.params(PushFeatureSwitchParams.OptoutExptPushCapParam) + ) + .apply(Seq(target)) + .map { values => + val isValid = values.headOption.getOrElse(false) + if (!isValid && isNewUser) { + statsReceiver.scope(name).counter("filtered_new_users").incr() + } + isValid + } + } else Future.True + case _ => Future.True + } + } + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + /** + * Predicate used to specify CRT fatigue given interval and max number of candidates within interval. + * @param crt The specific CRT that this predicate is being applied to + * @param intervalParam The fatigue interval + * @param maxInIntervalParam The max number of the given CRT's candidates that are acceptable + * in the interval + * @param stats StatsReceiver + * @return Target Predicate + */ + def pushRecTypeFatiguePredicate( + crt: CRT, + intervalParam: Param[Duration], + maxInIntervalParam: FSBoundedParam[Int], + stats: StatsReceiver + ): Predicate[Target] = + Predicate.fromAsync { target: Target => + val interval = target.params(intervalParam) + val maxIninterval = target.params(maxInIntervalParam) + CommonFatiguePredicate + .recTypeTargetFatiguePredicate( + interval = interval, + maxInInterval = maxIninterval, + recommendationType = crt, + notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, + minInterval = 30.minutes + )(stats.scope(s"${crt}_push_candidate_fatigue")).apply(Seq(target)).map(_.head) + } + + def inlineActionFatiguePredicate( + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[Target] = { + val name = "inline_action_fatigue" + val predicateRequests = statsReceiver.scope(name).counter("requests") + val targetIsInExpt = statsReceiver.scope(name).counter("target_in_expt") + val predicateEnabled = statsReceiver.scope(name).counter("enabled") + val predicateDisabled = statsReceiver.scope(name).counter("disabled") + val inlineFatigueDisabled = statsReceiver.scope(name).counter("inline_fatigue_disabled") + + Predicate + .fromAsync { target: Target => + predicateRequests.incr() + if (target.params(PushFeatureSwitchParams.TargetInInlineActionAppVisitFatigue)) { + targetIsInExpt.incr() + target.inlineActionHistory.map { inlineHistory => + if (inlineHistory.nonEmpty && target.params( + PushFeatureSwitchParams.EnableInlineActionAppVisitFatigue)) { + predicateEnabled.incr() + val inlineFatigue = target.params(PushFeatureSwitchParams.InlineActionAppVisitFatigue) + val lookbackInMs = inlineFatigue.ago.inMilliseconds + val filteredHistory = inlineHistory.filter { + case (time, _) => time > lookbackInMs + } + filteredHistory.isEmpty + } else { + inlineFatigueDisabled.incr() + true + } + } + } else { + predicateDisabled.incr() + Future.True + } + } + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + def webNotifsHoldback[T <: TargetUser with UserDetails with TargetABDecider]( + )( + implicit stats: StatsReceiver + ): NamedPredicate[T] = { + val name = "mr_web_notifs_holdback" + Predicate + .fromAsync { targetUserContext: T => + targetUserContext.deviceInfo.map { deviceInfoOpt => + val isPrimaryWeb = deviceInfoOpt.exists { + _.guessedPrimaryClient.exists { clientType => + clientType == MobileClientType.Web + } + } + !(isPrimaryWeb && targetUserContext.params(PushFeatureSwitchParams.MRWebHoldbackParam)) + } + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.scala new file mode 100644 index 0000000000..be5993b71b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TopTweetImpressionsPredicates.scala @@ -0,0 +1,56 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.model.TopTweetImpressionsPushCandidate +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate + +object TopTweetImpressionsPredicates { + + def topTweetImpressionsFatiguePredicate( + implicit stats: StatsReceiver + ): NamedPredicate[TopTweetImpressionsPushCandidate] = { + val name = "top_tweet_impressions_fatigue" + val scopedStats = stats.scope(name) + val bucketImpressionCounter = scopedStats.counter("bucket_impression_count") + Predicate + .fromAsync { candidate: TopTweetImpressionsPushCandidate => + val interval = candidate.target.params(FS.TopTweetImpressionsNotificationInterval) + val maxInInterval = candidate.target.params(FS.MaxTopTweetImpressionsNotifications) + val minInterval = candidate.target.params(FS.TopTweetImpressionsFatigueMinIntervalDuration) + bucketImpressionCounter.incr() + + val fatiguePredicate = FatiguePredicate.recTypeOnly( + interval = interval, + maxInInterval = maxInInterval, + minInterval = minInterval, + recommendationType = CommonRecommendationType.TweetImpressions + ) + fatiguePredicate.apply(Seq(candidate)).map(_.head) + } + .withStats(stats.scope(s"predicate_${name}")) + .withName(name) + } + + def topTweetImpressionsThreshold( + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[TopTweetImpressionsPushCandidate] = { + val name = "top_tweet_impressions_threshold" + val scopedStats = statsReceiver.scope(name) + val meetsImpressionsCounter = scopedStats.counter("meets_impressions_count") + val bucketImpressionCounter = scopedStats.counter("bucket_impression_count") + Predicate + .from[TopTweetImpressionsPushCandidate] { candidate => + val meetsImpressionsThreshold = + candidate.impressionsCount >= candidate.target.params(FS.TopTweetImpressionsThreshold) + if (meetsImpressionsThreshold) meetsImpressionsCounter.incr() + bucketImpressionCounter.incr() + meetsImpressionsThreshold + } + .withStats(statsReceiver.scope(s"predicate_${name}")) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.scala new file mode 100644 index 0000000000..d1a3a1c644 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetEngagementRatioPredicate.scala @@ -0,0 +1,112 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +object TweetEngagementRatioPredicate { + + def QTtoNtabClickBasedPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[ + PushCandidate with TweetCandidate with RecommendationType + ] = { + val name = "oon_tweet_engagement_filter_qt_to_ntabclick_ratio_based_predicate" + val scopedStatsReceiver = stats.scope(name) + val allOonCandidatesCounter = scopedStatsReceiver.counter("all_oon_candidates") + val filteredCandidatesCounter = + scopedStatsReceiver.counter("filtered_oon_candidates") + + val quoteCountFeature = + "tweet.core.tweet_counts.quote_count" + val ntabClickCountFeature = + "tweet.magic_recs_tweet_real_time_aggregates_v2.pair.v2.magicrecs.realtime.is_ntab_clicked.any_feature.Duration.Top.count" + + Predicate + .fromAsync { candidate: PushCandidate with TweetCandidate with RecommendationType => + val target = candidate.target + val crt = candidate.commonRecType + val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || + RecTypes.outOfNetworkTopicTweetTypes.contains(crt) + + lazy val QTtoNtabClickRatioThreshold = + target.params(PushFeatureSwitchParams.TweetQTtoNtabClickRatioThresholdParam) + lazy val quoteCount = candidate.numericFeatures.getOrElse(quoteCountFeature, 0.0) + lazy val ntabClickCount = candidate.numericFeatures.getOrElse(ntabClickCountFeature, 0.0) + lazy val quoteRate = if (ntabClickCount > 0) quoteCount / ntabClickCount else 1.0 + + if (isOonCandidate) allOonCandidatesCounter.incr() + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { + val ntabClickThreshold = 1000 + candidate.cachePredicateInfo( + name + "_count", + ntabClickCount, + ntabClickThreshold, + ntabClickCount >= ntabClickThreshold) + candidate.cachePredicateInfo( + name + "_ratio", + quoteRate, + QTtoNtabClickRatioThreshold, + quoteRate < QTtoNtabClickRatioThreshold) + if (ntabClickCount >= ntabClickThreshold && quoteRate < QTtoNtabClickRatioThreshold) { + filteredCandidatesCounter.incr() + Future.False + } else Future.True + } else Future.True + } + .withStats(stats.scope(name)) + .withName(name) + } + + def TweetReplyLikeRatioPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with TweetCandidate] = { + val name = "tweet_reply_like_ratio" + val scopedStatsReceiver = stats.scope(name) + val allCandidatesCounter = scopedStatsReceiver.counter("all_candidates") + val filteredCandidatesCounter = scopedStatsReceiver.counter("filtered_candidates") + val bucketedCandidatesCounter = scopedStatsReceiver.counter("bucketed_candidates") + + Predicate + .fromAsync { candidate: PushCandidate => + allCandidatesCounter.incr() + val target = candidate.target + val likeCount = candidate.numericFeatures + .getOrElse(PushConstants.TweetLikesFeatureName, 0.0) + val replyCount = candidate.numericFeatures + .getOrElse(PushConstants.TweetRepliesFeatureName, 0.0) + val ratio = replyCount / likeCount.max(1) + val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) || + RecTypes.outOfNetworkTopicTweetTypes.contains(candidate.commonRecType) + + if (isOonCandidate + && CandidateUtil.shouldApplyHealthQualityFilters(candidate) + && replyCount > target.params( + PushFeatureSwitchParams.TweetReplytoLikeRatioReplyCountThreshold)) { + bucketedCandidatesCounter.incr() + if (ratio > target.params( + PushFeatureSwitchParams.TweetReplytoLikeRatioThresholdLowerBound) + && ratio < target.params( + PushFeatureSwitchParams.TweetReplytoLikeRatioThresholdUpperBound)) { + filteredCandidatesCounter.incr() + Future.False + } else { + Future.True + } + } else { + Future.True + } + } + .withStats(stats.scope(s"predicate_$name")) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.scala new file mode 100644 index 0000000000..4ff24ae779 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetLanguagePredicate.scala @@ -0,0 +1,109 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.util.CandidateUtil +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.language.normalization.UserDisplayLanguage +import com.twitter.util.Future + +object TweetLanguagePredicate { + + def oonTweeetLanguageMatch( + )( + implicit stats: StatsReceiver + ): NamedPredicate[ + PushCandidate with RecommendationType with TweetDetails + ] = { + val name = "oon_tweet_language_predicate" + val scopedStatsReceiver = stats.scope(name) + val oonCandidatesCounter = + scopedStatsReceiver.counter("oon_candidates") + val enableFilterCounter = + scopedStatsReceiver.counter("enabled_filter") + val skipMediaTweetsCounter = + scopedStatsReceiver.counter("skip_media_tweets") + + Predicate + .fromAsync { candidate: PushCandidate with RecommendationType with TweetDetails => + val target = candidate.target + val crt = candidate.commonRecType + val isOonCandidate = RecTypes.isOutOfNetworkTweetRecType(crt) || + RecTypes.outOfNetworkTopicTweetTypes.contains(crt) + + if (CandidateUtil.shouldApplyHealthQualityFilters(candidate) && isOonCandidate) { + oonCandidatesCounter.incr() + + target.featureMap.map { featureMap => + val userPreferredLanguages = featureMap.sparseBinaryFeatures + .getOrElse("user.language.user.preferred_contents", Set.empty[String]) + val userEngagementLanguages = featureMap.sparseContinuousFeatures.getOrElse( + "user.language.user.engagements", + Map.empty[String, Double]) + val userFollowLanguages = featureMap.sparseContinuousFeatures.getOrElse( + "user.language.user.following_accounts", + Map.empty[String, Double]) + val userProducedTweetLanguages = featureMap.sparseContinuousFeatures + .getOrElse("user.language.user.produced_tweets", Map.empty) + val userDeviceLanguages = featureMap.sparseContinuousFeatures.getOrElse( + "user.language.user.recent_devices", + Map.empty[String, Double]) + val tweetLanguageOpt = candidate.categoricalFeatures + .get(target.params(PushFeatureSwitchParams.TweetLanguageFeatureNameParam)) + + if (userPreferredLanguages.isEmpty) + scopedStatsReceiver.counter("userPreferredLanguages_empty").incr() + if (userEngagementLanguages.isEmpty) + scopedStatsReceiver.counter("userEngagementLanguages_empty").incr() + if (userFollowLanguages.isEmpty) + scopedStatsReceiver.counter("userFollowLanguages_empty").incr() + if (userProducedTweetLanguages.isEmpty) + scopedStatsReceiver + .counter("userProducedTweetLanguages_empty") + .incr() + if (userDeviceLanguages.isEmpty) + scopedStatsReceiver.counter("userDeviceLanguages_empty").incr() + if (tweetLanguageOpt.isEmpty) scopedStatsReceiver.counter("tweetLanguage_empty").incr() + + val tweetLanguage = tweetLanguageOpt.getOrElse("und") + val undefinedTweetLanguages = Set("") + + if (!undefinedTweetLanguages.contains(tweetLanguage)) { + lazy val userInferredLanguageThreshold = + target.params(PushFeatureSwitchParams.UserInferredLanguageThresholdParam) + lazy val userDeviceLanguageThreshold = + target.params(PushFeatureSwitchParams.UserDeviceLanguageThresholdParam) + lazy val enableTweetLanguageFilter = + target.params(PushFeatureSwitchParams.EnableTweetLanguageFilter) + lazy val skipLanguageFilterForMediaTweets = + target.params(PushFeatureSwitchParams.SkipLanguageFilterForMediaTweets) + + lazy val allLanguages = userPreferredLanguages ++ + userEngagementLanguages.filter(_._2 > userInferredLanguageThreshold).keySet ++ + userFollowLanguages.filter(_._2 > userInferredLanguageThreshold).keySet ++ + userProducedTweetLanguages.filter(_._2 > userInferredLanguageThreshold).keySet ++ + userDeviceLanguages.filter(_._2 > userDeviceLanguageThreshold).keySet + + if (enableTweetLanguageFilter && allLanguages.nonEmpty) { + enableFilterCounter.incr() + val hasMedia = candidate.hasPhoto || candidate.hasVideo + + if (hasMedia && skipLanguageFilterForMediaTweets) { + skipMediaTweetsCounter.incr() + true + } else { + allLanguages.map(UserDisplayLanguage.toTweetLanguage).contains(tweetLanguage) + } + } else true + } else true + } + } else Future.True + } + .withStats(stats.scope(name)) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.scala new file mode 100644 index 0000000000..c055369092 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/TweetWithheldContentPredicate.scala @@ -0,0 +1,35 @@ +package com.twitter.frigate.pushservice.predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TweetDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.predicate.tweetypie.UserLocationAndTweet +import com.twitter.hermit.predicate.tweetypie.WithheldTweetPredicate +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.service.metastore.gen.thriftscala.Location +import com.twitter.util.Future + +object TweetWithheldContentPredicate { + val name = "withheld_content" + val defaultLocation = Location(city = "", region = "", countryCode = "", confidence = 0.0) + + def apply( + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with TweetDetails] = { + Predicate + .fromAsync { candidate: PushCandidate with TweetDetails => + candidate.tweet match { + case Some(tweet) => + WithheldTweetPredicate(checkAllCountries = true) + .apply(Seq(UserLocationAndTweet(defaultLocation, tweet))) + .map(_.head) + case None => + Future.value(false) + } + } + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.scala new file mode 100644 index 0000000000..86c1f8abd5 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/event/EventPredicatesForCandidate.scala @@ -0,0 +1,155 @@ +package com.twitter.frigate.pushservice.predicate.event + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.EventCandidate +import com.twitter.frigate.common.base.TargetInfo +import com.twitter.frigate.common.base.TargetUser +import com.twitter.frigate.common.candidate.FrigateHistory +import com.twitter.frigate.common.history.RecItems +import com.twitter.frigate.magic_events.thriftscala.Locale +import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutNewsEventPushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutPredicatesUtil._ +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +object EventPredicatesForCandidate { + def hasTitle( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { + val name = "event_title_available" + val scopedStatsReceiver = statsReceiver.scope(s"predicate_$name") + Predicate + .fromAsync { candidate: MagicFanoutEventHydratedCandidate => + candidate.eventTitleFut.map(_.nonEmpty) + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + def isNotDuplicateWithEventId( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { + val name = "duplicate_event_id" + Predicate + .fromAsync { candidate: MagicFanoutEventHydratedCandidate => + val useRelaxedFatigueLengthFut: Future[Boolean] = + candidate match { + case mfNewsEvent: MagicFanoutNewsEventPushCandidate => + mfNewsEvent.isHighPriorityEvent + case _ => Future.value(false) + } + Future.join(candidate.target.history, useRelaxedFatigueLengthFut).map { + case (history, useRelaxedFatigueLength) => + val filteredNotifications = if (useRelaxedFatigueLength) { + val relaxedFatigueInterval = + candidate.target + .params( + PushFeatureSwitchParams.MagicFanoutRelaxedEventIdFatigueIntervalInHours).hours + history.notificationMap.filterKeys { time => + time.untilNow <= relaxedFatigueInterval + }.values + } else history.notificationMap.values + !RecItems(filteredNotifications.toSeq).events.exists(_.eventId == candidate.eventId) + } + } + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } + + def isNotDuplicateWithEventIdForCandidate[ + T <: TargetUser with FrigateHistory, + Cand <: EventCandidate with TargetInfo[T] + ]( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[Cand] = { + val name = "is_not_duplicate_event" + Predicate + .fromAsync { candidate: Cand => + candidate.target.pushRecItems.map { + !_.events.map(_.eventId).contains(candidate.eventId) + } + } + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + def accountCountryPredicateWithAllowlist( + implicit stats: StatsReceiver + ): NamedPredicate[MagicFanoutEventPushCandidate] = { + val name = "account_country_predicate_with_allowlist" + val scopedStats = stats.scope(name) + + val skipPredicate = Predicate + .from { candidate: MagicFanoutEventPushCandidate => + candidate.target.params(PushFeatureSwitchParams.MagicFanoutSkipAccountCountryPredicate) + } + .withStats(stats.scope("skip_account_country_predicate_mf")) + .withName("skip_account_country_predicate_mf") + + val excludeEventFromAccountCountryPredicateFiltering = Predicate + .from { candidate: MagicFanoutEventPushCandidate => + val eventId = candidate.eventId + val target = candidate.target + target + .params(PushFeatureSwitchParams.MagicFanoutEventAllowlistToSkipAccountCountryPredicate) + .exists(eventId.equals) + } + .withStats(stats.scope("exclude_event_from_account_country_predicate_filtering")) + .withName("exclude_event_from_account_country_predicate_filtering") + + skipPredicate + .or(excludeEventFromAccountCountryPredicateFiltering) + .or(accountCountryPredicate) + .withStats(scopedStats) + .withName(name) + } + + /** + * Check if user's country is targeted + * @param stats + */ + def accountCountryPredicate( + implicit stats: StatsReceiver + ): NamedPredicate[MagicFanoutEventPushCandidate] = { + val name = "account_country_predicate" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + val internationalLocalePassedCounter = + scopedStatsReceiver.counter("international_locale_passed") + val internationalLocaleFilteredCounter = + scopedStatsReceiver.counter("international_locale_filtered") + Predicate + .fromAsync { candidate: MagicFanoutEventPushCandidate => + candidate.target.countryCode.map { + case Some(countryCode) => + val denyListedCountryCodes: Seq[String] = + if (candidate.commonRecType == CommonRecommendationType.MagicFanoutNewsEvent) { + candidate.target + .params(PushFeatureSwitchParams.MagicFanoutDenyListedCountries) + } else if (candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent) { + candidate.target + .params(PushFeatureSwitchParams.MagicFanoutSportsEventDenyListedCountries) + } else Seq() + val eventCountries = + candidate.newsForYouMetadata + .flatMap(_.locales).getOrElse(Seq.empty[Locale]).flatMap(_.country) + if (isInCountryList(countryCode, eventCountries) + && !isInCountryList(countryCode, denyListedCountryCodes)) { + internationalLocalePassedCounter.incr() + true + } else { + internationalLocaleFilteredCounter.incr() + false + } + case _ => false + } + } + .withStats(scopedStatsReceiver) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.scala new file mode 100644 index 0000000000..52371b4881 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesForCandidate.scala @@ -0,0 +1,525 @@ +package com.twitter.frigate.pushservice.predicate.magic_fanout + +import com.twitter.audience_rewards.thriftscala.HasSuperFollowingRelationshipRequest +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.MagicFanoutCandidate +import com.twitter.frigate.common.base.MagicFanoutCreatorEventCandidate +import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate +import com.twitter.frigate.common.history.RecItems +import com.twitter.frigate.common.predicate.FatiguePredicate.build +import com.twitter.frigate.common.predicate.FatiguePredicate.productLaunchTypeRecTypesOnlyFilter +import com.twitter.frigate.common.predicate.FatiguePredicate.recOnlyFilter +import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext +import com.twitter.frigate.common.store.interests.SemanticCoreEntityId +import com.twitter.frigate.common.util.IbisAppPushDeviceSettingsUtil +import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType +import com.twitter.frigate.magic_events.thriftscala.ProductType +import com.twitter.frigate.magic_events.thriftscala.TargetID +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutEventHydratedCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutNewsEventPushCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.predicate.FatiguePredicate +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.NotificationDisplayLocation +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.interests.thriftscala.UserInterests +import com.twitter.simclusters_v2.thriftscala.EmbeddingType +import com.twitter.simclusters_v2.thriftscala.ModelVersion +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.configapi.Param +import com.twitter.util.Duration +import com.twitter.util.Future + +object MagicFanoutPredicatesForCandidate { + + /** + * Check if Semantic Core reasons satisfy rank threshold ( for heavy users a non broad entity should satisfy the threshold) + */ + def magicFanoutErgInterestRankThresholdPredicate( + implicit stats: StatsReceiver + ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { + val name = "magicfanout_interest_erg_rank_threshold" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + Predicate + .fromAsync { candidate: MagicFanoutEventHydratedCandidate => + candidate.target.isHeavyUserState.map { isHeavyUser => + lazy val rankThreshold = + if (isHeavyUser) { + candidate.target.params(PushFeatureSwitchParams.MagicFanoutRankErgThresholdHeavy) + } else { + candidate.target.params(PushFeatureSwitchParams.MagicFanoutRankErgThresholdNonHeavy) + } + MagicFanoutPredicatesUtil + .checkIfValidErgScEntityReasonExists( + candidate.effectiveMagicEventsReasons, + rankThreshold + ) + } + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + def newsNotificationFatigue( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val name = "news_notification_fatigue" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + Predicate + .fromAsync { candidate: PushCandidate => + FatiguePredicate + .recTypeSetOnly( + notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, + recTypes = Set(CommonRecommendationType.MagicFanoutNewsEvent), + maxInInterval = + candidate.target.params(PushFeatureSwitchParams.MFMaxNumberOfPushesInInterval), + interval = candidate.target.params(PushFeatureSwitchParams.MFPushIntervalInHours), + minInterval = candidate.target.params(PushFeatureSwitchParams.MFMinIntervalFatigue) + ) + .apply(Seq(candidate)) + .map(_.headOption.getOrElse(false)) + + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + /** + * Check if reason contains any optouted semantic core entity interests. + * + * @param stats + * + * @return + */ + def magicFanoutNoOptoutInterestPredicate( + implicit stats: StatsReceiver + ): NamedPredicate[MagicFanoutEventPushCandidate] = { + val name = "magicfanout_optout_interest_predicate" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + val withOptOutInterestsCounter = stats.counter("with_optout_interests") + val withoutOptOutInterestsCounter = stats.counter("without_optout_interests") + Predicate + .fromAsync { candidate: MagicFanoutEventPushCandidate => + candidate.target.optOutSemanticCoreInterests.map { + case ( + optOutUserInterests: Seq[SemanticCoreEntityId] + ) => + withOptOutInterestsCounter.incr() + optOutUserInterests + .intersect(candidate.annotatedAndInferredSemanticCoreEntities).isEmpty + case _ => + withoutOptOutInterestsCounter.incr() + true + } + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + /** + * Checks if the target has only one device language language, + * and that language is targeted for that event + * + * @param statsReceiver + * + * @return + */ + def inferredUserDeviceLanguagePredicate( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[MagicFanoutEventPushCandidate] = { + val name = "inferred_device_language" + val scopedStats = statsReceiver.scope(s"predicate_$name") + Predicate + .fromAsync { candidate: MagicFanoutEventPushCandidate => + val target = candidate.target + target.deviceInfo.map { + _.flatMap { deviceInfo => + val languages = deviceInfo.deviceLanguages.getOrElse(Seq.empty[String]) + val distinctDeviceLanguages = + IbisAppPushDeviceSettingsUtil.distinctDeviceLanguages(languages) + + candidate.newsForYouMetadata.map { newsForYouMetadata => + val eventLocales = newsForYouMetadata.locales.getOrElse(Seq.empty) + val eventLanguages = eventLocales.flatMap(_.language).map(_.toLowerCase).distinct + + eventLanguages.intersect(distinctDeviceLanguages).nonEmpty + } + }.getOrElse(false) + } + } + .withStats(scopedStats) + .withName(name) + } + + /** + * Bypass predicate if high priority push + */ + def highPriorityNewsEventExceptedPredicate( + predicate: NamedPredicate[MagicFanoutNewsEventPushCandidate] + )( + implicit config: Config + ): NamedPredicate[MagicFanoutNewsEventPushCandidate] = { + PredicatesForCandidate.exceptedPredicate( + name = "high_priority_excepted_" + predicate.name, + fn = MagicFanoutPredicatesUtil.checkIfHighPriorityNewsEventForCandidate, + predicate + )(config.statsReceiver) + } + + /** + * Bypass predicate if high priority push + */ + def highPriorityEventExceptedPredicate( + predicate: NamedPredicate[MagicFanoutEventPushCandidate] + )( + implicit config: Config + ): NamedPredicate[MagicFanoutEventPushCandidate] = { + PredicatesForCandidate.exceptedPredicate( + name = "high_priority_excepted_" + predicate.name, + fn = MagicFanoutPredicatesUtil.checkIfHighPriorityEventForCandidate, + predicate + )(config.statsReceiver) + } + + def magicFanoutSimClusterTargetingPredicate( + implicit stats: StatsReceiver + ): NamedPredicate[MagicFanoutEventPushCandidate] = { + val name = "simcluster_targeting" + val scopedStats = stats.scope(s"predicate_$name") + val userStateCounters = scopedStats.scope("user_state") + Predicate + .fromAsync { candidate: MagicFanoutEventPushCandidate => + candidate.target.isHeavyUserState.map { isHeavyUser => + val simClusterEmbeddings = candidate.newsForYouMetadata.flatMap( + _.eventContextScribe.flatMap(_.simClustersEmbeddings)) + val TopKSimClustersCount = 50 + val eventSimClusterVectorOpt: Option[MagicFanoutPredicatesUtil.SimClusterScores] = + MagicFanoutPredicatesUtil.getEventSimClusterVector( + simClusterEmbeddings.map(_.toMap), + (ModelVersion.Model20m145kUpdated, EmbeddingType.FollowBasedTweet), + TopKSimClustersCount + ) + val userSimClusterVectorOpt: Option[MagicFanoutPredicatesUtil.SimClusterScores] = + MagicFanoutPredicatesUtil.getUserSimClusterVector(candidate.effectiveMagicEventsReasons) + (eventSimClusterVectorOpt, userSimClusterVectorOpt) match { + case ( + Some(eventSimClusterVector: MagicFanoutPredicatesUtil.SimClusterScores), + Some(userSimClusterVector)) => + val score = eventSimClusterVector + .normedDotProduct(userSimClusterVector, eventSimClusterVector) + val threshold = if (isHeavyUser) { + candidate.target.params( + PushFeatureSwitchParams.MagicFanoutSimClusterDotProductHeavyUserThreshold) + } else { + candidate.target.params( + PushFeatureSwitchParams.MagicFanoutSimClusterDotProductNonHeavyUserThreshold) + } + val isPassed = score >= threshold + userStateCounters.scope(isHeavyUser.toString).counter(s"$isPassed").incr() + isPassed + + case (None, Some(userSimClusterVector)) => + candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent + + case _ => false + } + } + } + .withStats(scopedStats) + .withName(name) + } + + def geoTargetingHoldback( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with MagicFanoutCandidate] = { + Predicate + .from[PushCandidate with MagicFanoutCandidate] { candidate => + if (MagicFanoutPredicatesUtil.reasonsContainGeoTarget( + candidate.candidateMagicEventsReasons)) { + candidate.target.params(PushFeatureSwitchParams.EnableMfGeoTargeting) + } else true + } + .withStats(stats.scope("geo_targeting_holdback")) + .withName("geo_targeting_holdback") + } + + def geoOptOutPredicate( + userStore: ReadableStore[Long, User] + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with MagicFanoutCandidate] = { + Predicate + .fromAsync[PushCandidate with MagicFanoutCandidate] { candidate => + if (MagicFanoutPredicatesUtil.reasonsContainGeoTarget( + candidate.candidateMagicEventsReasons)) { + userStore.get(candidate.target.targetId).map { userOpt => + val isGeoAllowed = userOpt + .flatMap(_.account) + .exists(_.allowLocationHistoryPersonalization) + isGeoAllowed + } + } else { + Future.True + } + } + .withStats(stats.scope("geo_opt_out_predicate")) + .withName("geo_opt_out_predicate") + } + + /** + * Check if Semantic Core reasons contains valid utt reason & reason is within top k topics followed by user + */ + def magicFanoutTopicFollowsTargetingPredicate( + implicit stats: StatsReceiver, + interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests] + ): NamedPredicate[MagicFanoutEventHydratedCandidate] = { + val name = "magicfanout_topic_follows_targeting" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + Predicate + .fromAsync[PushCandidate with MagicFanoutEventHydratedCandidate] { candidate => + candidate.followedTopicLocalizedEntities.map(_.nonEmpty) + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + /** Requires the magicfanout candidate to have a UserID reason which ranks below the follow + * rank threshold. If no UserID target exists the candidate is dropped. */ + def followRankThreshold( + threshold: Param[Int] + )( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with MagicFanoutCandidate] = { + val name = "follow_rank_threshold" + Predicate + .from[PushCandidate with MagicFanoutCandidate] { c => + c.candidateMagicEventsReasons.exists { fanoutReason => + fanoutReason.reason match { + case TargetID.UserID(_) => + fanoutReason.rank.exists { rank => + rank <= c.target.params(threshold) + } + case _ => false + } + } + } + .withStats(statsReceiver.scope(name)) + .withName(name) + } + + def userGeneratedEventsPredicate( + implicit statsReceiver: StatsReceiver + ): NamedPredicate[PushCandidate with MagicFanoutEventHydratedCandidate] = { + val name = "user_generated_moments" + val stats = statsReceiver.scope(name) + + Predicate + .from { candidate: PushCandidate with MagicFanoutEventHydratedCandidate => + val isUgmMoment = candidate.semanticCoreEntityTags.values.flatten.toSet + .contains(MagicFanoutPredicatesUtil.UgmMomentTag) + if (isUgmMoment) { + candidate.target.params(PushFeatureSwitchParams.MagicFanoutNewsUserGeneratedEventsEnable) + } else true + }.withStats(stats) + .withName(name) + } + def escherbirdMagicfanoutEventParam( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with MagicFanoutEventPushCandidate] = { + val name = "magicfanout_escherbird_fs" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + + Predicate + .fromAsync[PushCandidate with MagicFanoutEventPushCandidate] { candidate => + val candidateFrigateNotif = candidate.frigateNotification.magicFanoutEventNotification + val isEscherbirdEvent = candidateFrigateNotif.exists(_.isEscherbirdEvent.contains(true)) + scopedStatsReceiver.counter(s"with_escherbird_flag_$isEscherbirdEvent").incr() + + if (isEscherbirdEvent) { + + val listOfEventsSemanticCoreDomainIds = + candidate.target.params(PushFeatureSwitchParams.ListOfEventSemanticCoreDomainIds) + + val candScDomainEvent = + if (listOfEventsSemanticCoreDomainIds.nonEmpty) { + candidate.eventSemanticCoreDomainIds + .intersect(listOfEventsSemanticCoreDomainIds).nonEmpty + } else { + false + } + scopedStatsReceiver + .counter( + s"with_escherbird_fs_in_list_of_event_semantic_core_domains_$candScDomainEvent").incr() + Future.value(candScDomainEvent) + } else { + Future.True + } + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + /** + * Checks if the user has custom targeting enabled.If so, bucket the user in experiment. This custom targeting refers to adding + * tweet authors as targets in the eventfanout service. + * @param stats [StatsReceiver] + * @return NamedPredicate[PushCandidate with MagicFanoutEventPushCandidate] + */ + def hasCustomTargetingForNewsEventsParam( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with MagicFanoutEventPushCandidate] = { + val name = "magicfanout_hascustomtargeting" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + + Predicate + .from[PushCandidate with MagicFanoutEventPushCandidate] { candidate => + candidate.candidateMagicEventsReasons.exists { fanoutReason => + fanoutReason.reason match { + case userIdReason: TargetID.UserID => + if (userIdReason.userID.hasCustomTargeting.contains(true)) { + candidate.target.params( + PushFeatureSwitchParams.MagicFanoutEnableCustomTargetingNewsEvent) + } else true + case _ => true + } + } + } + .withStats(scopedStatsReceiver) + .withName(name) + + } + + def magicFanoutProductLaunchFatigue( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with MagicFanoutProductLaunchCandidate] = { + val name = "magic_fanout_product_launch_fatigue" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + Predicate + .fromAsync { candidate: PushCandidate with MagicFanoutProductLaunchCandidate => + val target = candidate.target + val (interval, maxInInterval, minInterval) = { + candidate.productLaunchType match { + case ProductType.BlueVerified => + ( + target.params(PushFeatureSwitchParams.ProductLaunchPushIntervalInHours), + target.params(PushFeatureSwitchParams.ProductLaunchMaxNumberOfPushesInInterval), + target.params(PushFeatureSwitchParams.ProductLaunchMinIntervalFatigue)) + case _ => + (Duration.fromDays(1), 0, Duration.Zero) + } + } + build( + interval = interval, + maxInInterval = maxInInterval, + minInterval = minInterval, + filterHistory = productLaunchTypeRecTypesOnlyFilter( + Set(CommonRecommendationType.MagicFanoutProductLaunch), + candidate.productLaunchType.toString), + notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice + ).flatContraMap { candidate: PushCandidate => candidate.target.history } + .apply(Seq(candidate)) + .map(_.headOption.getOrElse(false)) + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + def creatorPushTargetIsNotCreator( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { + val name = "magic_fanout_creator_is_self" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + Predicate + .from { candidate: PushCandidate with MagicFanoutCreatorEventCandidate => + candidate.target.targetId != candidate.creatorId + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + def duplicateCreatorPredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { + val name = "magic_fanout_creator_duplicate_creator_id" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + Predicate + .fromAsync { cand: PushCandidate with MagicFanoutCreatorEventCandidate => + cand.target.pushRecItems.map { recItems: RecItems => + !recItems.creatorIds.contains(cand.creatorId) + } + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + def isSuperFollowingCreator( + )( + implicit config: Config, + stats: StatsReceiver + ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { + val name = "magic_fanout_is_already_superfollowing_creator" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + Predicate + .fromAsync { cand: PushCandidate with MagicFanoutCreatorEventCandidate => + config.hasSuperFollowingRelationshipStore + .get( + HasSuperFollowingRelationshipRequest( + sourceUserId = cand.target.targetId, + targetUserId = cand.creatorId)).map(_.getOrElse(false)) + } + .withStats(scopedStatsReceiver) + .withName(name) + } + + def magicFanoutCreatorPushFatiguePredicate( + )( + implicit stats: StatsReceiver + ): NamedPredicate[PushCandidate with MagicFanoutCreatorEventCandidate] = { + val name = "magic_fanout_creator_fatigue" + val scopedStatsReceiver = stats.scope(s"predicate_$name") + Predicate + .fromAsync { candidate: PushCandidate with MagicFanoutCreatorEventCandidate => + val target = candidate.target + val (interval, maxInInterval, minInterval) = { + candidate.creatorFanoutType match { + case CreatorFanoutType.UserSubscription => + ( + target.params(PushFeatureSwitchParams.CreatorSubscriptionPushIntervalInHours), + target.params( + PushFeatureSwitchParams.CreatorSubscriptionPushMaxNumberOfPushesInInterval), + target.params(PushFeatureSwitchParams.CreatorSubscriptionPushhMinIntervalFatigue)) + case CreatorFanoutType.NewCreator => + ( + target.params(PushFeatureSwitchParams.NewCreatorPushIntervalInHours), + target.params(PushFeatureSwitchParams.NewCreatorPushMaxNumberOfPushesInInterval), + target.params(PushFeatureSwitchParams.NewCreatorPushMinIntervalFatigue)) + case _ => + (Duration.fromDays(1), 0, Duration.Zero) + } + } + build( + interval = interval, + maxInInterval = maxInInterval, + minInterval = minInterval, + filterHistory = recOnlyFilter(candidate.commonRecType), + notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice + ).flatContraMap { candidate: PushCandidate => candidate.target.history } + .apply(Seq(candidate)) + .map(_.headOption.getOrElse(false)) + } + .withStats(scopedStatsReceiver) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.scala new file mode 100644 index 0000000000..306f2b3b68 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutPredicatesUtil.scala @@ -0,0 +1,218 @@ +package com.twitter.frigate.pushservice.predicate.magic_fanout + +import com.twitter.eventdetection.event_context.util.SimClustersUtil +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.magic_events.thriftscala._ +import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutNewsEventPushCandidate +import com.twitter.frigate.pushservice.model.MagicFanoutProductLaunchPushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.simclusters_v2.common.SimClustersEmbedding +import com.twitter.simclusters_v2.thriftscala.EmbeddingType +import com.twitter.simclusters_v2.thriftscala.ModelVersion +import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId +import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding} +import com.twitter.util.Future + +object MagicFanoutPredicatesUtil { + + val UttDomain: Long = 0L + type DomainId = Long + type EntityId = Long + val BroadCategoryTag = "utt:broad_category" + val UgmMomentTag = "MMTS.isUGMMoment" + val TopKSimClustersCount = 50 + + case class SimClusterScores(simClusterScoreVector: Map[Int, Double]) { + def dotProduct(other: SimClusterScores): Double = { + simClusterScoreVector + .map { + case (clusterId, score) => other.simClusterScoreVector.getOrElse(clusterId, 0.0) * score + }.foldLeft(0.0) { _ + _ } + } + + def norm(): Double = { + val sumOfSquares: Double = simClusterScoreVector + .map { + case (clusterId, score) => score * score + }.foldLeft(0.0)(_ + _) + scala.math.sqrt(sumOfSquares) + } + + def normedDotProduct(other: SimClusterScores, normalizer: SimClusterScores): Double = { + val denominator = normalizer.norm() + val score = dotProduct(other) + if (denominator != 0.0) { + score / denominator + } else { + score + } + } + } + + private def isSemanticCoreEntityBroad( + semanticCoreEntityTags: Map[(DomainId, EntityId), Set[String]], + scEntityId: SemanticCoreID + ): Boolean = { + semanticCoreEntityTags + .getOrElse((scEntityId.domainId, scEntityId.entityId), Set.empty).contains(BroadCategoryTag) + } + + def isInCountryList(accountCountryCode: String, locales: Seq[String]): Boolean = { + locales.map(_.toLowerCase).contains(accountCountryCode.toLowerCase) + } + + /** + * Boolean check of if a MagicFanout is high priority push + */ + def checkIfHighPriorityNewsEventForCandidate( + candidate: MagicFanoutNewsEventPushCandidate + ): Future[Boolean] = { + candidate.isHighPriorityEvent.map { isHighPriority => + isHighPriority && (candidate.target.params(PushFeatureSwitchParams.EnableHighPriorityPush)) + } + } + + /** + * Boolean check of if a MagicFanout event is high priority push + */ + def checkIfHighPriorityEventForCandidate( + candidate: MagicFanoutEventPushCandidate + ): Future[Boolean] = { + candidate.isHighPriorityEvent.map { isHighPriority => + candidate.commonRecType match { + case CommonRecommendationType.MagicFanoutSportsEvent => + isHighPriority && (candidate.target.params( + PushFeatureSwitchParams.EnableHighPrioritySportsPush)) + case _ => false + } + } + } + + /** + * Boolean check if to skip target blue verified + */ + def shouldSkipBlueVerifiedCheckForCandidate( + candidate: MagicFanoutProductLaunchPushCandidate + ): Future[Boolean] = + Future.value( + candidate.target.params(PushFeatureSwitchParams.DisableIsTargetBlueVerifiedPredicate)) + + /** + * Boolean check if to skip target is legacy verified + */ + def shouldSkipLegacyVerifiedCheckForCandidate( + candidate: MagicFanoutProductLaunchPushCandidate + ): Future[Boolean] = + Future.value( + candidate.target.params(PushFeatureSwitchParams.DisableIsTargetLegacyVerifiedPredicate)) + + def shouldSkipSuperFollowCreatorCheckForCandidate( + candidate: MagicFanoutProductLaunchPushCandidate + ): Future[Boolean] = + Future.value( + !candidate.target.params(PushFeatureSwitchParams.EnableIsTargetSuperFollowCreatorPredicate)) + + /** + * Boolean check of if a reason of a MagicFanout is higher than the rank threshold of an event + */ + def checkIfErgScEntityReasonMeetsThreshold( + rankThreshold: Int, + reason: MagicEventsReason, + ): Boolean = { + reason.reason match { + case TargetID.SemanticCoreID(scEntityId: SemanticCoreID) => + reason.rank match { + case Some(rank) => rank < rankThreshold + case _ => false + } + case _ => false + } + } + + /** + * Check if MagicEventsReasons contains a reason that matches the thresholdw + */ + def checkIfValidErgScEntityReasonExists( + magicEventsReasons: Option[Seq[MagicEventsReason]], + rankThreshold: Int + )( + implicit stats: StatsReceiver + ): Boolean = { + magicEventsReasons match { + case Some(reasons) if reasons.exists(_.isNewUser.contains(true)) => true + case Some(reasons) => + reasons.exists { reason => + reason.source.contains(ReasonSource.ErgShortTermInterestSemanticCore) && + checkIfErgScEntityReasonMeetsThreshold( + rankThreshold, + reason + ) + } + + case _ => false + } + } + + /** + * Get event simcluster vector from event context + */ + def getEventSimClusterVector( + simClustersEmbeddingOption: Option[Map[SimClustersEmbeddingId, ThriftSimClustersEmbedding]], + embeddingMapKey: (ModelVersion, EmbeddingType), + topKSimClustersCount: Int + ): Option[SimClusterScores] = { + simClustersEmbeddingOption.map { thriftSimClustersEmbeddings => + val simClustersEmbeddings: Map[SimClustersEmbeddingId, SimClustersEmbedding] = + thriftSimClustersEmbeddings.map { + case (simClustersEmbeddingId, simClustersEmbeddingValue) => + (simClustersEmbeddingId, SimClustersEmbedding(simClustersEmbeddingValue)) + }.toMap + val emptySeq = Seq[(Int, Double)]() + val simClusterScoreTuple: Map[(ModelVersion, EmbeddingType), Seq[(Int, Double)]] = + SimClustersUtil + .getMaxTopKTweetSimClusters(simClustersEmbeddings, topKSimClustersCount) + SimClusterScores(simClusterScoreTuple.getOrElse(embeddingMapKey, emptySeq).toMap) + } + } + + /** + * Get user simcluster vector magic events reasons + */ + def getUserSimClusterVector( + magicEventsReasonsOpt: Option[Seq[MagicEventsReason]] + ): Option[SimClusterScores] = { + magicEventsReasonsOpt.map { magicEventsReasons: Seq[MagicEventsReason] => + val reasons: Seq[(Int, Double)] = magicEventsReasons.flatMap { reason => + reason.reason match { + case TargetID.SimClusterID(simClusterId: SimClusterID) => + Some((simClusterId.clusterId, reason.score.getOrElse(0.0))) + case _ => + None + } + } + SimClusterScores(reasons.toMap) + } + } + + def reasonsContainGeoTarget(reasons: Seq[MagicEventsReason]): Boolean = { + reasons.exists { reason => + val isGeoGraphSource = reason.source.contains(ReasonSource.GeoGraph) + reason.reason match { + case TargetID.PlaceID(_) if isGeoGraphSource => true + case _ => false + } + } + } + + def geoPlaceIdsFromReasons(reasons: Seq[MagicEventsReason]): Set[Long] = { + reasons.flatMap { reason => + val isGeoGraphSource = reason.source.contains(ReasonSource.GeoGraph) + reason.reason match { + case TargetID.PlaceID(PlaceID(id)) if isGeoGraphSource => Some(id) + case _ => None + } + }.toSet + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.scala new file mode 100644 index 0000000000..224be3ad5e --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutSportsUtil.scala @@ -0,0 +1,231 @@ +package com.twitter.frigate.pushservice.predicate.magic_fanout + +import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate +import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate +import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerPeriod +import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventHomeAwayTeamScore +import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventStatus +import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventTeamAlignment.Away +import com.twitter.datatools.entityservice.entities.sports.thriftscala.SportsEventTeamAlignment.Home +import com.twitter.escherbird.metadata.thriftscala.EntityMegadata +import com.twitter.frigate.pushservice.params.SportGameEnum +import com.twitter.frigate.common.base.GenericGameScore +import com.twitter.frigate.common.base.NflGameScore +import com.twitter.frigate.common.base.SoccerGameScore +import com.twitter.frigate.common.base.TeamInfo +import com.twitter.frigate.common.base.TeamScore +import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +object MagicFanoutSportsUtil { + + def transformSoccerGameScore(game: SoccerMatchLiveUpdate): Option[SoccerGameScore] = { + require(game.status.isDefined) + val gameScore = transformToGameScore(game.score, game.status.get) + val _penaltyKicks = transformToGameScore(game.penaltyScore, game.status.get) + gameScore.map { score => + val _isGameEnd = game.status.get match { + case SportsEventStatus.Completed(_) => true + case _ => false + } + + val _isHalfTime = game.period.exists { period => + period match { + case SoccerPeriod.Halftime(_) => true + case _ => false + } + } + + val _isOvertime = game.period.exists { period => + period match { + case SoccerPeriod.PreOvertime(_) => true + case _ => false + } + } + + val _isPenaltyKicks = game.period.exists { period => + period match { + case SoccerPeriod.PrePenalty(_) => true + case SoccerPeriod.Penalty(_) => true + case _ => false + } + } + + val _gameMinute = game.gameMinute.map { soccerGameMinute => + game.minutesInInjuryTime match { + case Some(injuryTime) => s"($soccerGameMinute+$injuryTime′)" + case None => s"($soccerGameMinute′)" + } + } + + SoccerGameScore( + score.home, + score.away, + isGameOngoing = score.isGameOngoing, + penaltyKicks = _penaltyKicks, + gameMinute = _gameMinute, + isHalfTime = _isHalfTime, + isOvertime = _isOvertime, + isPenaltyKicks = _isPenaltyKicks, + isGameEnd = _isGameEnd + ) + } + } + + def transformNFLGameScore(game: NflFootballGameLiveUpdate): Option[NflGameScore] = { + require(game.status.isDefined) + + val gameScore = transformToGameScore(game.score, game.status.get) + gameScore.map { score => + val _isGameEnd = game.status.get match { + case SportsEventStatus.Completed(_) => true + case _ => false + } + + val _matchTime = (game.quarter, game.remainingSecondsInQuarter) match { + case (Some(quarter), Some(remainingSeconds)) if remainingSeconds != 0L => + val m = (remainingSeconds / 60) % 60 + val s = remainingSeconds % 60 + val formattedSeconds = "%02d:%02d".format(m, s) + s"(Q$quarter - $formattedSeconds)" + case (Some(quarter), None) => s"(Q$quarter)" + case _ => "" + } + + NflGameScore( + score.home, + score.away, + isGameOngoing = score.isGameOngoing, + isGameEnd = _isGameEnd, + matchTime = _matchTime + ) + } + } + + /** + Takes a score from Strato columns and turns it into an easier to handle structure (GameScore class) + We do this to easily access the home/away scenario for copy setting + */ + def transformToGameScore( + scoreOpt: Option[SportsEventHomeAwayTeamScore], + status: SportsEventStatus + ): Option[GenericGameScore] = { + val isGameOngoing = status match { + case SportsEventStatus.InProgress(_) => true + case SportsEventStatus.Completed(_) => false + case _ => false + } + + val scoresWithTeam = scoreOpt + .map { score => + score.scores.map { score => (score.score, score.participantAlignment, score.participantId) } + }.getOrElse(Seq()) + + val tuple = scoresWithTeam match { + case Seq(teamOne, teamTwo, _*) => Some((teamOne, teamTwo)) + case _ => None + } + tuple.flatMap { + case ((Some(teamOneScore), teamOneAlignment, teamOne), (Some(teamTwoScore), _, teamTwo)) => + teamOneAlignment.flatMap { + case Home(_) => + val home = TeamScore(teamOneScore, teamOne.entityId, teamOne.domainId) + val away = TeamScore(teamTwoScore, teamTwo.entityId, teamTwo.domainId) + Some(GenericGameScore(home, away, isGameOngoing)) + case Away(_) => + val away = TeamScore(teamOneScore, teamOne.entityId, teamOne.domainId) + val home = TeamScore(teamTwoScore, teamTwo.entityId, teamTwo.domainId) + Some(GenericGameScore(home, away, isGameOngoing)) + case _ => None + } + case _ => None + } + } + + def getTeamInfo( + team: TeamScore, + semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata] + ): Future[Option[TeamInfo]] = { + semanticCoreMegadataStore + .get(SemanticEntityForQuery(team.teamDomainId, team.teamEntityId)).map { + _.flatMap { + _.basicMetadata.map { metadata => + TeamInfo( + name = metadata.name, + twitterUserId = metadata.twitter.flatMap(_.preferredTwitterUserId)) + } + } + } + } + + def getNFLReadableName(name: String): String = { + val teamNames = + Seq("") + teamNames.find(teamName => name.contains(teamName)).getOrElse(name) + } + + def getSoccerIbisMap(game: SoccerGameScore): Map[String, String] = { + val gameMinuteMap = game.gameMinute + .map { gameMinute => Map("match_time" -> gameMinute) } + .getOrElse(Map.empty) + + val updateTypeMap = { + if (game.isGameEnd) Map("is_game_end" -> "true") + else if (game.isHalfTime) Map("is_half_time" -> "true") + else if (game.isOvertime) Map("is_overtime" -> "true") + else if (game.isPenaltyKicks) Map("is_penalty_kicks" -> "true") + else Map("is_score_update" -> "true") + } + + val awayScore = game match { + case SoccerGameScore(_, away, _, None, _, _, _, _, _) => + away.score.toString + case SoccerGameScore(_, away, _, Some(penaltyKick), _, _, _, _, _) => + s"${away.score} (${penaltyKick.away.score}) " + case _ => "" + } + + val homeScore = game match { + case SoccerGameScore(home, _, _, None, _, _, _, _, _) => + home.score.toString + case SoccerGameScore(home, _, _, Some(penaltyKick), _, _, _, _, _) => + s"${home.score} (${penaltyKick.home.score}) " + case _ => "" + } + + val scoresMap = Map( + "away_score" -> awayScore, + "home_score" -> homeScore, + ) + + gameType(SportGameEnum.Soccer) ++ updateTypeMap ++ gameMinuteMap ++ scoresMap + } + + def getNflIbisMap(game: NflGameScore): Map[String, String] = { + val gameMinuteMap = Map("match_time" -> game.matchTime) + + val updateTypeMap = { + if (game.isGameEnd) Map("is_game_end" -> "true") + else Map("is_score_update" -> "true") + } + + val awayScore = game.away.score + val homeScore = game.home.score + + val scoresMap = Map( + "away_score" -> awayScore.toString, + "home_score" -> homeScore.toString, + ) + + gameType(SportGameEnum.Nfl) ++ updateTypeMap ++ gameMinuteMap ++ scoresMap + } + + private def gameType(game: SportGameEnum.Value): Map[String, String] = { + game match { + case SportGameEnum.Soccer => Map("is_soccer_game" -> "true") + case SportGameEnum.Nfl => Map("is_nfl_game" -> "true") + case _ => Map.empty + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.scala new file mode 100644 index 0000000000..758c9ef34b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/magic_fanout/MagicFanoutTargetingPredicateWrappersForCandidate.scala @@ -0,0 +1,133 @@ +package com.twitter.frigate.pushservice.predicate.magic_fanout + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext +import com.twitter.frigate.common.util.FeatureSwitchParams +import com.twitter.frigate.common.util.MagicFanoutTargetingPredicatesEnum +import com.twitter.frigate.common.util.MagicFanoutTargetingPredicatesEnum.MagicFanoutTargetingPredicatesEnum +import com.twitter.frigate.pushservice.model.MagicFanoutEventPushCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.interests.thriftscala.UserInterests +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.configapi.FSEnumParam + +object MagicFanoutTargetingPredicateWrappersForCandidate { + + /** + * Combine Prod and Experimental Targeting predicate logic + * @return: NamedPredicate[MagicFanoutNewsEventPushCandidate] + */ + def magicFanoutTargetingPredicate( + stats: StatsReceiver, + config: Config + ): NamedPredicate[MagicFanoutEventPushCandidate] = { + val name = "magic_fanout_targeting_predicate" + Predicate + .fromAsync { candidate: MagicFanoutEventPushCandidate => + val mfTargetingPredicateParam = getTargetingPredicateParams(candidate) + val mfTargetingPredicate = MagicFanoutTargetingPredicateMapForCandidate + .apply(config) + .get(candidate.target.params(mfTargetingPredicateParam)) + mfTargetingPredicate match { + case Some(predicate) => + predicate.apply(Seq(candidate)).map(_.head) + case None => + throw new Exception( + s"MFTargetingPredicateMap doesnt contain value for TargetingParam: ${FeatureSwitchParams.MFTargetingPredicate}") + } + } + .withStats(stats.scope(name)) + .withName(name) + } + + private def getTargetingPredicateParams( + candidate: MagicFanoutEventPushCandidate + ): FSEnumParam[MagicFanoutTargetingPredicatesEnum.type] = { + if (candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent) { + FeatureSwitchParams.MFCricketTargetingPredicate + } else FeatureSwitchParams.MFTargetingPredicate + } + + /** + * SimCluster and ERG and Topic Follows Targeting Predicate + */ + def simClusterErgTopicFollowsTargetingPredicate( + implicit stats: StatsReceiver, + interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests] + ): NamedPredicate[MagicFanoutEventPushCandidate] = { + simClusterErgTargetingPredicate + .or(MagicFanoutPredicatesForCandidate.magicFanoutTopicFollowsTargetingPredicate) + .withName("sim_cluster_erg_topic_follows_targeting") + } + + /** + * SimCluster and ERG and Topic Follows Targeting Predicate + */ + def simClusterErgTopicFollowsUserFollowsTargetingPredicate( + implicit stats: StatsReceiver, + interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests] + ): NamedPredicate[MagicFanoutEventPushCandidate] = { + simClusterErgTopicFollowsTargetingPredicate + .or( + MagicFanoutPredicatesForCandidate.followRankThreshold( + PushFeatureSwitchParams.MagicFanoutRealgraphRankThreshold)) + .withName("sim_cluster_erg_topic_follows_user_follows_targeting") + } + + /** + * SimCluster and ERG Targeting Predicate + */ + def simClusterErgTargetingPredicate( + implicit stats: StatsReceiver + ): NamedPredicate[MagicFanoutEventPushCandidate] = { + MagicFanoutPredicatesForCandidate.magicFanoutSimClusterTargetingPredicate + .or(MagicFanoutPredicatesForCandidate.magicFanoutErgInterestRankThresholdPredicate) + .withName("sim_cluster_erg_targeting") + } +} + +/** + * Object to initalze and get predicate map + */ +object MagicFanoutTargetingPredicateMapForCandidate { + + /** + * Called from the Config.scala at the time of server initialization + * @param statsReceiver: implict stats receiver + * @return Map[MagicFanoutTargetingPredicatesEnum, NamedPredicate[MagicFanoutNewsEventPushCandidate]] + */ + def apply( + config: Config + ): Map[MagicFanoutTargetingPredicatesEnum, NamedPredicate[MagicFanoutEventPushCandidate]] = { + Map( + MagicFanoutTargetingPredicatesEnum.SimClusterAndERGAndTopicFollows -> MagicFanoutTargetingPredicateWrappersForCandidate + .simClusterErgTopicFollowsTargetingPredicate( + config.statsReceiver, + config.interestsWithLookupContextStore), + MagicFanoutTargetingPredicatesEnum.SimClusterAndERG -> MagicFanoutTargetingPredicateWrappersForCandidate + .simClusterErgTargetingPredicate(config.statsReceiver), + MagicFanoutTargetingPredicatesEnum.SimCluster -> MagicFanoutPredicatesForCandidate + .magicFanoutSimClusterTargetingPredicate(config.statsReceiver), + MagicFanoutTargetingPredicatesEnum.ERG -> MagicFanoutPredicatesForCandidate + .magicFanoutErgInterestRankThresholdPredicate(config.statsReceiver), + MagicFanoutTargetingPredicatesEnum.TopicFollows -> MagicFanoutPredicatesForCandidate + .magicFanoutTopicFollowsTargetingPredicate( + config.statsReceiver, + config.interestsWithLookupContextStore), + MagicFanoutTargetingPredicatesEnum.UserFollows -> MagicFanoutPredicatesForCandidate + .followRankThreshold( + PushFeatureSwitchParams.MagicFanoutRealgraphRankThreshold + )(config.statsReceiver), + MagicFanoutTargetingPredicatesEnum.SimClusterAndERGAndTopicFollowsAndUserFollows -> + MagicFanoutTargetingPredicateWrappersForCandidate + .simClusterErgTopicFollowsUserFollowsTargetingPredicate( + config.statsReceiver, + config.interestsWithLookupContextStore + ) + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.scala new file mode 100644 index 0000000000..704f300a5c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/CRTBasedNtabCaretClickFatiguePredicates.scala @@ -0,0 +1,973 @@ +package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.notificationservice.thriftscala.GenericType +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue +import com.twitter.hermit.predicate.Predicate +import com.twitter.frigate.common.base.Candidate +import com.twitter.frigate.common.base.RecommendationType +import com.twitter.frigate.common.base.TargetInfo +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.SeeLessOftenType +import com.twitter.frigate.common.history.History +import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate.TimeSeries +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper +import com.twitter.frigate.pushservice.predicate.CaretFeedbackHistoryFilter +import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.frigate.common.predicate.FatiguePredicate +import com.twitter.frigate.pushservice.util.PushCapUtil +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.util.PushDeviceUtil + +object CRTBasedNtabCaretClickFatiguePredicates { + + private val MagicRecsCategory = "MagicRecs" + + private val HighQualityRefreshableTypes: Set[Option[String]] = Set( + Some("MagicRecHighQualityTweet"), + ) + + private def getUserStateWeight(target: Target): Future[Double] = { + PushDeviceUtil.isNtabOnlyEligible.map { + case true => + target.params(PushFeatureSwitchParams.SeeLessOftenNtabOnlyNotifUserPushCapWeight) + case _ => 1.0 + } + } + + def crtToSeeLessOftenType( + crt: CommonRecommendationType, + candidate: Candidate + with RecommendationType + with TargetInfo[ + Target + ], + ): SeeLessOftenType = { + val crtToSeeLessOftenTypeMap: Map[CommonRecommendationType, SeeLessOftenType] = { + RecTypes.f1FirstDegreeTypes.map((_, SeeLessOftenType.F1Type)).toMap + } + + crtToSeeLessOftenTypeMap.getOrElse(crt, SeeLessOftenType.OtherTypes) + } + + def genericTypeToSeeLessOftenType( + genericType: GenericType, + candidate: Candidate + with RecommendationType + with TargetInfo[ + Target + ] + ): SeeLessOftenType = { + val genericTypeToSeeLessOftenTypeMap: Map[GenericType, SeeLessOftenType] = { + Map(GenericType.MagicRecFirstDegreeTweetRecent -> SeeLessOftenType.F1Type) + } + + genericTypeToSeeLessOftenTypeMap.getOrElse(genericType, SeeLessOftenType.OtherTypes) + } + + def getWeightForCaretFeedback( + dislikedType: SeeLessOftenType, + candidate: Candidate + with RecommendationType + with TargetInfo[ + Target + ] + ): Double = { + def getWeightFromDislikedAndCurrentType( + dislikedType: SeeLessOftenType, + currentType: SeeLessOftenType + ): Double = { + val weightMap: Map[(SeeLessOftenType, SeeLessOftenType), Double] = { + + Map( + (SeeLessOftenType.F1Type, SeeLessOftenType.F1Type) -> candidate.target.params( + PushFeatureSwitchParams.SeeLessOftenF1TriggerF1PushCapWeight), + (SeeLessOftenType.OtherTypes, SeeLessOftenType.OtherTypes) -> candidate.target.params( + PushFeatureSwitchParams.SeeLessOftenNonF1TriggerNonF1PushCapWeight), + (SeeLessOftenType.F1Type, SeeLessOftenType.OtherTypes) -> candidate.target.params( + PushFeatureSwitchParams.SeeLessOftenF1TriggerNonF1PushCapWeight), + (SeeLessOftenType.OtherTypes, SeeLessOftenType.F1Type) -> candidate.target.params( + PushFeatureSwitchParams.SeeLessOftenNonF1TriggerF1PushCapWeight) + ) + } + + weightMap + .getOrElse( + (dislikedType, currentType), + candidate.target.params(PushFeatureSwitchParams.SeeLessOftenDefaultPushCapWeight)) + } + + getWeightFromDislikedAndCurrentType( + dislikedType, + crtToSeeLessOftenType(candidate.commonRecType, candidate)) + } + + private def isOutsideCrtBasedNtabCaretClickFatiguePeriodContFn( + candidate: Candidate + with RecommendationType + with TargetInfo[ + Target + ], + history: History, + feedbackDetails: Seq[CaretFeedbackDetails], + filterHistory: TimeSeries => TimeSeries = + FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), + filterCaretFeedbackHistory: Target => Seq[ + CaretFeedbackDetails + ] => Seq[CaretFeedbackDetails] = + CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), + knobs: Seq[Double], + pushCapKnobs: Seq[Double], + powerKnobs: Seq[Double], + f1Weight: Double, + nonF1Weight: Double, + defaultPushCap: Int, + stats: StatsReceiver, + tripHqTweetWeight: Double = 0.0, + ): Boolean = { + val filteredFeedbackDetails = filterCaretFeedbackHistory(candidate.target)(feedbackDetails) + val weight = { + if (RecTypes.HighQualityTweetTypes.contains( + candidate.commonRecType) && (tripHqTweetWeight != 0)) { + tripHqTweetWeight + } else if (RecTypes.isF1Type(candidate.commonRecType)) { + f1Weight + } else { + nonF1Weight + } + } + val filteredHistory = History(filterHistory(history.history.toSeq).toMap) + isOutsideFatiguePeriod( + filteredHistory, + filteredFeedbackDetails, + Seq(), + ContinuousFunctionParam( + knobs, + pushCapKnobs, + powerKnobs, + weight, + defaultPushCap + ), + stats.scope( + if (RecTypes.isF1Type(candidate.commonRecType)) "mr_ntab_dislike_f1_candidate_fn" + else if (RecTypes.HighQualityTweetTypes.contains(candidate.commonRecType)) + "mr_ntab_dislike_high_quality_candidate_fn" + else "mr_ntab_dislike_nonf1_candidate_fn") + ) + } + + private def isOutsideFatiguePeriod( + history: History, + feedbackDetails: Seq[CaretFeedbackDetails], + feedbacks: Seq[FeedbackModel], + param: ContinuousFunctionParam, + stats: StatsReceiver + ): Boolean = { + val fatiguePeriod: Duration = + NtabCaretClickFatigueUtils.durationToFilterForFeedback( + feedbackDetails, + feedbacks, + param, + param.defaultValue, + stats + ) + + val hasRecentSent = + NtabCaretClickFatiguePredicateHelper.hasRecentSend(history, fatiguePeriod) + !hasRecentSent + + } + + def genericCRTBasedNtabCaretClickFnFatiguePredicate[ + Cand <: Candidate with RecommendationType with TargetInfo[ + Target + ] + ]( + filterHistory: TimeSeries => TimeSeries = + FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), + filterCaretFeedbackHistory: Target => Seq[ + CaretFeedbackDetails + ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter + .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), + filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = + NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) + )( + implicit stats: StatsReceiver + ): NamedPredicate[Cand] = { + val predicateName = "generic_crt_based_ntab_dislike_fatigue_fn" + Predicate + .fromAsync[Cand] { cand: Cand => + { + if (!cand.target.params(PushFeatureSwitchParams.EnableGenericCRTBasedFatiguePredicate)) { + Future.True + } else { + val scopedStats = stats.scope(predicateName) + val totalRequests = scopedStats.counter("mr_ntab_dislike_total") + val total90Day = + scopedStats.counter("mr_ntab_dislike_90day_dislike") + val totalDisabled = + scopedStats.counter("mr_ntab_dislike_not_90day_dislike") + val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") + val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") + val totalWithHistory = + scopedStats.counter("mr_ntab_dislike_with_history") + val totalWithoutHistory = + scopedStats.counter("mr_ntab_dislike_without_history") + totalRequests.incr() + + Future + .join( + cand.target.history, + cand.target.caretFeedbacks, + cand.target.dynamicPushcap, + cand.target.optoutAdjustedPushcap, + PushCapUtil.getDefaultPushCap(cand.target), + getUserStateWeight(cand.target) + ).map { + case ( + history, + Some(feedbackDetails), + dynamicPushcapOpt, + optoutAdjustedPushcapOpt, + defaultPushCap, + userStateWeight) => { + totalWithHistory.incr() + + val feedbackDetailsDeduped = + NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( + filterCaretFeedbackHistory(cand.target)(feedbackDetails), + stats + ) + + val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { + case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap + case (Some(pushcapInfo), _) => pushcapInfo.pushcap + case _ => defaultPushCap + } + val filteredHistory = History(filterHistory(history.history.toSeq).toMap) + + val hasUserDislikeInLast90Days = + NtabCaretClickFatigueUtils.hasUserDislikeInLast90Days(feedbackDetailsDeduped) + val isF1TriggerFatigueEnabled = cand.target + .params(PushFeatureSwitchParams.EnableContFnF1TriggerSeeLessOftenFatigue) + val isNonF1TriggerFatigueEnabled = cand.target.params( + PushFeatureSwitchParams.EnableContFnNonF1TriggerSeeLessOftenFatigue) + + val isOutisdeSeeLessOftenFatigue = + if (hasUserDislikeInLast90Days && (isF1TriggerFatigueEnabled || isNonF1TriggerFatigueEnabled)) { + total90Day.incr() + + val feedbackDetailsGroupedBySeeLessOftenType: Map[Option[ + SeeLessOftenType + ], Seq[ + CaretFeedbackDetails + ]] = feedbackDetails.groupBy(feedbackDetail => + feedbackDetail.genericNotificationMetadata.map(x => + genericTypeToSeeLessOftenType(x.genericType, cand))) + + val isOutsideFatiguePeriodSeq = + for (elem <- feedbackDetailsGroupedBySeeLessOftenType if elem._1.isDefined) + yield { + val dislikedSeeLessOftenType: SeeLessOftenType = elem._1.get + val seqCaretFeedbackDetails: Seq[CaretFeedbackDetails] = elem._2 + + val weight = getWeightForCaretFeedback( + dislikedSeeLessOftenType, + cand) * userStateWeight + + if (isOutsideFatiguePeriod( + history = filteredHistory, + feedbackDetails = seqCaretFeedbackDetails, + feedbacks = Seq(), + param = ContinuousFunctionParam( + knobs = cand.target + .params(PushFeatureSwitchParams.SeeLessOftenListOfDayKnobs), + knobValues = cand.target + .params( + PushFeatureSwitchParams.SeeLessOftenListOfPushCapWeightKnobs).map( + _ * pushCap), + powers = cand.target + .params(PushFeatureSwitchParams.SeeLessOftenListOfPowerKnobs), + weight = weight, + defaultValue = pushCap + ), + scopedStats + )) { + true + } else { + false + } + } + + isOutsideFatiguePeriodSeq.forall(identity) + } else { + totalDisabled.incr() + true + } + + if (isOutisdeSeeLessOftenFatigue) { + totalSuccess.incr() + } else totalFiltered.incr() + + isOutisdeSeeLessOftenFatigue + } + + case _ => + totalSuccess.incr() + totalWithoutHistory.incr() + true + } + } + } + }.withStats(stats.scope(predicateName)) + .withName(predicateName) + } + + def f1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ + Cand <: Candidate with RecommendationType with TargetInfo[ + Target + ] + ]( + filterHistory: TimeSeries => TimeSeries = + FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), + filterCaretFeedbackHistory: Target => Seq[ + CaretFeedbackDetails + ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter + .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), + filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = + NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) + )( + implicit stats: StatsReceiver + ): NamedPredicate[Cand] = { + val predicateName = "f1_triggered_crt_based_ntab_dislike_fatigue_fn" + Predicate + .fromAsync[Cand] { cand: Cand => + { + val scopedStats = stats.scope(predicateName) + val totalRequests = scopedStats.counter("mr_ntab_dislike_total") + val total90Day = + scopedStats.counter("mr_ntab_dislike_90day_dislike") + val totalDisabled = + scopedStats.counter("mr_ntab_dislike_not_90day_dislike") + val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") + val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") + val totalWithHistory = + scopedStats.counter("mr_ntab_dislike_with_history") + val totalWithoutHistory = + scopedStats.counter("mr_ntab_dislike_without_history") + totalRequests.incr() + + Future + .join( + cand.target.history, + cand.target.caretFeedbacks, + cand.target.dynamicPushcap, + cand.target.optoutAdjustedPushcap, + cand.target.notificationFeedbacks, + PushCapUtil.getDefaultPushCap(cand.target), + getUserStateWeight(cand.target) + ).map { + case ( + history, + Some(feedbackDetails), + dynamicPushcapOpt, + optoutAdjustedPushcapOpt, + Some(feedbacks), + defaultPushCap, + userStateWeight) => + totalWithHistory.incr() + + val feedbackDetailsDeduped = + NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( + filterCaretFeedbackHistory(cand.target)(feedbackDetails), + stats + ) + + val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { + case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap + case (Some(pushcapInfo), _) => pushcapInfo.pushcap + case _ => defaultPushCap + } + val filteredHistory = History(filterHistory(history.history.toSeq).toMap) + + val isOutsideInlineDislikeFatigue = + if (cand.target + .params(PushFeatureSwitchParams.EnableContFnF1TriggerInlineFeedbackFatigue)) { + val weight = + if (RecTypes.isF1Type(cand.commonRecType)) { + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackF1TriggerF1PushCapWeight) + } else { + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackF1TriggerNonF1PushCapWeight) + } + + val inlineFeedbackFatigueParam = ContinuousFunctionParam( + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackListOfDayKnobs), + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackListOfPushCapWeightKnobs) + .map(_ * pushCap), + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackListOfPowerKnobs), + weight, + pushCap + ) + + isInlineDislikeOutsideFatiguePeriod( + cand, + feedbacks + .collect { + case feedbackPromptValue: FeedbackPromptValue => + InlineFeedbackModel(feedbackPromptValue, None) + }, + filteredHistory, + Seq( + filterInlineFeedbackHistory, + NtabCaretClickFatigueUtils.feedbackModelFilterByCRT( + RecTypes.f1FirstDegreeTypes)), + inlineFeedbackFatigueParam, + scopedStats + ) + } else true + + lazy val isOutsidePromptDislikeFatigue = + if (cand.target + .params(PushFeatureSwitchParams.EnableContFnF1TriggerPromptFeedbackFatigue)) { + val weight = + if (RecTypes.isF1Type(cand.commonRecType)) { + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackF1TriggerF1PushCapWeight) + } else { + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackF1TriggerNonF1PushCapWeight) + } + + val promptFeedbackFatigueParam = ContinuousFunctionParam( + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackListOfDayKnobs), + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackListOfPushCapWeightKnobs) + .map(_ * pushCap), + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackListOfPowerKnobs), + weight, + pushCap + ) + + isPromptDislikeOutsideFatiguePeriod( + feedbacks + .collect { + case feedbackPromptValue: FeedbackPromptValue => + PromptFeedbackModel(feedbackPromptValue, None) + }, + filteredHistory, + Seq( + filterInlineFeedbackHistory, + NtabCaretClickFatigueUtils.feedbackModelFilterByCRT( + RecTypes.f1FirstDegreeTypes)), + promptFeedbackFatigueParam, + scopedStats + ) + } else true + + isOutsideInlineDislikeFatigue && isOutsidePromptDislikeFatigue + + case _ => + totalSuccess.incr() + totalWithoutHistory.incr() + true + } + } + }.withStats(stats.scope(predicateName)) + .withName(predicateName) + } + + def nonF1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ + Cand <: Candidate with RecommendationType with TargetInfo[ + Target + ] + ]( + filterHistory: TimeSeries => TimeSeries = + FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), + filterCaretFeedbackHistory: Target => Seq[ + CaretFeedbackDetails + ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter + .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), + filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = + NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) + )( + implicit stats: StatsReceiver + ): NamedPredicate[Cand] = { + val predicateName = "non_f1_triggered_crt_based_ntab_dislike_fatigue_fn" + Predicate + .fromAsync[Cand] { cand: Cand => + { + val scopedStats = stats.scope(predicateName) + val totalRequests = scopedStats.counter("mr_ntab_dislike_total") + val total90Day = + scopedStats.counter("mr_ntab_dislike_90day_dislike") + val totalDisabled = + scopedStats.counter("mr_ntab_dislike_not_90day_dislike") + val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") + val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") + val totalWithHistory = + scopedStats.counter("mr_ntab_dislike_with_history") + val totalWithoutHistory = + scopedStats.counter("mr_ntab_dislike_without_history") + val totalFeedbackSuccess = scopedStats.counter("mr_total_feedback_success") + totalRequests.incr() + + Future + .join( + cand.target.history, + cand.target.caretFeedbacks, + cand.target.dynamicPushcap, + cand.target.optoutAdjustedPushcap, + cand.target.notificationFeedbacks, + PushCapUtil.getDefaultPushCap(cand.target), + getUserStateWeight(cand.target), + ).map { + case ( + history, + Some(feedbackDetails), + dynamicPushcapOpt, + optoutAdjustedPushcapOpt, + Some(feedbacks), + defaultPushCap, + userStateWeight) => + totalWithHistory.incr() + + val filteredfeedbackDetails = + if (cand.target.params( + PushFeatureSwitchParams.AdjustTripHqTweetTriggeredNtabCaretClickFatigue)) { + val refreshableTypeFilter = CaretFeedbackHistoryFilter + .caretFeedbackHistoryFilterByRefreshableTypeDenyList( + HighQualityRefreshableTypes) + refreshableTypeFilter(cand.target)(feedbackDetails) + } else { + feedbackDetails + } + + val feedbackDetailsDeduped = + NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( + filterCaretFeedbackHistory(cand.target)(filteredfeedbackDetails), + stats + ) + + val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { + case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap + case (Some(pushcapInfo), _) => pushcapInfo.pushcap + case _ => defaultPushCap + } + val filteredHistory = History(filterHistory(history.history.toSeq).toMap) + + val isOutsideInlineDislikeFatigue = + if (cand.target + .params( + PushFeatureSwitchParams.EnableContFnNonF1TriggerInlineFeedbackFatigue)) { + val weight = + if (RecTypes.isF1Type(cand.commonRecType)) + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackNonF1TriggerF1PushCapWeight) + else + cand.target + .params( + PushFeatureSwitchParams.InlineFeedbackNonF1TriggerNonF1PushCapWeight) + + val inlineFeedbackFatigueParam = ContinuousFunctionParam( + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackListOfDayKnobs), + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackListOfPushCapWeightKnobs) + .map(_ * pushCap), + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackListOfPowerKnobs), + weight, + pushCap + ) + + val excludedCRTs: Set[CommonRecommendationType] = + if (cand.target.params( + PushFeatureSwitchParams.AdjustTripHqTweetTriggeredNtabCaretClickFatigue)) { + RecTypes.f1FirstDegreeTypes ++ RecTypes.HighQualityTweetTypes + } else { + RecTypes.f1FirstDegreeTypes + } + + isInlineDislikeOutsideFatiguePeriod( + cand, + feedbacks + .collect { + case feedbackPromptValue: FeedbackPromptValue => + InlineFeedbackModel(feedbackPromptValue, None) + }, + filteredHistory, + Seq( + filterInlineFeedbackHistory, + NtabCaretClickFatigueUtils.feedbackModelExcludeCRT(excludedCRTs)), + inlineFeedbackFatigueParam, + scopedStats + ) + } else true + + lazy val isOutsidePromptDislikeFatigue = + if (cand.target + .params( + PushFeatureSwitchParams.EnableContFnNonF1TriggerPromptFeedbackFatigue)) { + val weight = + if (RecTypes.isF1Type(cand.commonRecType)) + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackNonF1TriggerF1PushCapWeight) + else + cand.target + .params( + PushFeatureSwitchParams.PromptFeedbackNonF1TriggerNonF1PushCapWeight) + + val promptFeedbackFatigueParam = ContinuousFunctionParam( + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackListOfDayKnobs), + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackListOfPushCapWeightKnobs) + .map(_ * pushCap), + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackListOfPowerKnobs), + weight, + pushCap + ) + + isPromptDislikeOutsideFatiguePeriod( + feedbacks + .collect { + case feedbackPromptValue: FeedbackPromptValue => + PromptFeedbackModel(feedbackPromptValue, None) + }, + filteredHistory, + Seq( + filterInlineFeedbackHistory, + NtabCaretClickFatigueUtils.feedbackModelExcludeCRT( + RecTypes.f1FirstDegreeTypes)), + promptFeedbackFatigueParam, + scopedStats + ) + } else true + + isOutsideInlineDislikeFatigue && isOutsidePromptDislikeFatigue + case _ => + totalFeedbackSuccess.incr() + totalWithoutHistory.incr() + true + } + } + }.withStats(stats.scope(predicateName)) + .withName(predicateName) + } + + def tripHqTweetTriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ + Cand <: Candidate with RecommendationType with TargetInfo[ + Target + ] + ]( + filterHistory: TimeSeries => TimeSeries = + FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), + filterCaretFeedbackHistory: Target => Seq[ + CaretFeedbackDetails + ] => Seq[CaretFeedbackDetails] = CaretFeedbackHistoryFilter + .caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), + filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = + NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes) + )( + implicit stats: StatsReceiver + ): NamedPredicate[Cand] = { + val predicateName = "trip_hq_tweet_triggered_crt_based_ntab_dislike_fatigue_fn" + Predicate + .fromAsync[Cand] { cand: Cand => + { + val scopedStats = stats.scope(predicateName) + val totalRequests = scopedStats.counter("mr_ntab_dislike_total") + val total90Day = + scopedStats.counter("mr_ntab_dislike_90day_dislike") + val totalDisabled = + scopedStats.counter("mr_ntab_dislike_not_90day_dislike") + val totalSuccess = scopedStats.counter("mr_ntab_dislike_success") + val totalFiltered = scopedStats.counter("mr_ntab_dislike_filtered") + val totalWithHistory = + scopedStats.counter("mr_ntab_dislike_with_history") + val totalWithoutHistory = + scopedStats.counter("mr_ntab_dislike_without_history") + val totalFeedbackSuccess = scopedStats.counter("mr_total_feedback_success") + totalRequests.incr() + + Future + .join( + cand.target.history, + cand.target.caretFeedbacks, + cand.target.dynamicPushcap, + cand.target.optoutAdjustedPushcap, + cand.target.notificationFeedbacks, + PushCapUtil.getDefaultPushCap(cand.target), + getUserStateWeight(cand.target), + ).map { + case ( + history, + Some(feedbackDetails), + dynamicPushcapOpt, + optoutAdjustedPushcapOpt, + Some(feedbacks), + defaultPushCap, + userStateWeight) => + totalWithHistory.incr() + if (cand.target.params( + PushFeatureSwitchParams.AdjustTripHqTweetTriggeredNtabCaretClickFatigue)) { + + val refreshableTypeFilter = CaretFeedbackHistoryFilter + .caretFeedbackHistoryFilterByRefreshableType(HighQualityRefreshableTypes) + val filteredfeedbackDetails = refreshableTypeFilter(cand.target)(feedbackDetails) + + val feedbackDetailsDeduped = + NtabCaretClickFatiguePredicateHelper.dedupFeedbackDetails( + filterCaretFeedbackHistory(cand.target)(filteredfeedbackDetails), + stats + ) + + val pushCap: Int = (dynamicPushcapOpt, optoutAdjustedPushcapOpt) match { + case (_, Some(optoutAdjustedPushcap)) => optoutAdjustedPushcap + case (Some(pushcapInfo), _) => pushcapInfo.pushcap + case _ => defaultPushCap + } + val filteredHistory = History(filterHistory(history.history.toSeq).toMap) + + val isOutsideInlineDislikeFatigue = + if (cand.target + .params( + PushFeatureSwitchParams.EnableContFnNonF1TriggerInlineFeedbackFatigue)) { + val weight = { + if (RecTypes.HighQualityTweetTypes.contains(cand.commonRecType)) { + cand.target + .params( + PushFeatureSwitchParams.InlineFeedbackNonF1TriggerNonF1PushCapWeight) + } else { + cand.target + .params( + PushFeatureSwitchParams.InlineFeedbackNonF1TriggerF1PushCapWeight) + } + } + + val inlineFeedbackFatigueParam = ContinuousFunctionParam( + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackListOfDayKnobs), + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackListOfPushCapWeightKnobs) + .map(_ * pushCap), + cand.target + .params(PushFeatureSwitchParams.InlineFeedbackListOfPowerKnobs), + weight, + pushCap + ) + + val includedCRTs: Set[CommonRecommendationType] = + RecTypes.HighQualityTweetTypes + + isInlineDislikeOutsideFatiguePeriod( + cand, + feedbacks + .collect { + case feedbackPromptValue: FeedbackPromptValue => + InlineFeedbackModel(feedbackPromptValue, None) + }, + filteredHistory, + Seq( + filterInlineFeedbackHistory, + NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(includedCRTs)), + inlineFeedbackFatigueParam, + scopedStats + ) + } else true + + lazy val isOutsidePromptDislikeFatigue = + if (cand.target + .params( + PushFeatureSwitchParams.EnableContFnNonF1TriggerPromptFeedbackFatigue)) { + val weight = + if (RecTypes.isF1Type(cand.commonRecType)) + cand.target + .params( + PushFeatureSwitchParams.PromptFeedbackNonF1TriggerF1PushCapWeight) + else + cand.target + .params( + PushFeatureSwitchParams.PromptFeedbackNonF1TriggerNonF1PushCapWeight) + + val promptFeedbackFatigueParam = ContinuousFunctionParam( + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackListOfDayKnobs), + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackListOfPushCapWeightKnobs) + .map(_ * pushCap), + cand.target + .params(PushFeatureSwitchParams.PromptFeedbackListOfPowerKnobs), + weight, + pushCap + ) + + isPromptDislikeOutsideFatiguePeriod( + feedbacks + .collect { + case feedbackPromptValue: FeedbackPromptValue => + PromptFeedbackModel(feedbackPromptValue, None) + }, + filteredHistory, + Seq( + filterInlineFeedbackHistory, + NtabCaretClickFatigueUtils.feedbackModelExcludeCRT( + RecTypes.f1FirstDegreeTypes)), + promptFeedbackFatigueParam, + scopedStats + ) + } else true + + isOutsideInlineDislikeFatigue && isOutsidePromptDislikeFatigue + } else { + true + } + case _ => + totalFeedbackSuccess.incr() + totalWithoutHistory.incr() + true + } + } + }.withStats(stats.scope(predicateName)) + .withName(predicateName) + } + + private def getDedupedInlineFeedbackByType( + inlineFeedbacks: Seq[FeedbackModel], + feedbackType: FeedbackTypeEnum.Value, + revertedFeedbackType: FeedbackTypeEnum.Value + ): Seq[FeedbackModel] = { + inlineFeedbacks + .filter(feedback => + feedback.feedbackTypeEnum == feedbackType || + feedback.feedbackTypeEnum == revertedFeedbackType) + .groupBy(feedback => feedback.notificationImpressionId.getOrElse("")) + .toSeq + .collect { + case (impressionId, feedbacks: Seq[FeedbackModel]) if (feedbacks.nonEmpty) => + val latestFeedback = feedbacks.maxBy(feedback => feedback.timestampMs) + if (latestFeedback.feedbackTypeEnum == feedbackType) + Some(latestFeedback) + else None + case _ => None + } + .flatten + } + + private def getDedupedInlineFeedback( + inlineFeedbacks: Seq[FeedbackModel], + target: Target + ): Seq[FeedbackModel] = { + val inlineDislikeFeedback = + if (target.params(PushFeatureSwitchParams.UseInlineDislikeForFatigue)) { + getDedupedInlineFeedbackByType( + inlineFeedbacks, + FeedbackTypeEnum.InlineDislike, + FeedbackTypeEnum.InlineRevertedDislike) + } else Seq() + val inlineDismissFeedback = + if (target.params(PushFeatureSwitchParams.UseInlineDismissForFatigue)) { + getDedupedInlineFeedbackByType( + inlineFeedbacks, + FeedbackTypeEnum.InlineDismiss, + FeedbackTypeEnum.InlineRevertedDismiss) + } else Seq() + val inlineSeeLessFeedback = + if (target.params(PushFeatureSwitchParams.UseInlineSeeLessForFatigue)) { + getDedupedInlineFeedbackByType( + inlineFeedbacks, + FeedbackTypeEnum.InlineSeeLess, + FeedbackTypeEnum.InlineRevertedSeeLess) + } else Seq() + val inlineNotRelevantFeedback = + if (target.params(PushFeatureSwitchParams.UseInlineNotRelevantForFatigue)) { + getDedupedInlineFeedbackByType( + inlineFeedbacks, + FeedbackTypeEnum.InlineNotRelevant, + FeedbackTypeEnum.InlineRevertedNotRelevant) + } else Seq() + + inlineDislikeFeedback ++ inlineDismissFeedback ++ inlineSeeLessFeedback ++ inlineNotRelevantFeedback + } + + private def isInlineDislikeOutsideFatiguePeriod( + candidate: Candidate + with RecommendationType + with TargetInfo[ + Target + ], + inlineFeedbacks: Seq[FeedbackModel], + filteredHistory: History, + feedbackFilters: Seq[Seq[FeedbackModel] => Seq[FeedbackModel]], + inlineFeedbackFatigueParam: ContinuousFunctionParam, + stats: StatsReceiver + ): Boolean = { + val scopedStats = stats.scope("inline_dislike_fatigue") + + val inlineNegativeFeedback = + getDedupedInlineFeedback(inlineFeedbacks, candidate.target) + + val hydratedInlineNegativeFeedback = FeedbackModelHydrator.HydrateNotification( + inlineNegativeFeedback, + filteredHistory.history.toSeq.map(_._2)) + + if (isOutsideFatiguePeriod( + filteredHistory, + Seq(), + feedbackFilters.foldLeft(hydratedInlineNegativeFeedback)((feedbacks, feedbackFilter) => + feedbackFilter(feedbacks)), + inlineFeedbackFatigueParam, + scopedStats + )) { + scopedStats.counter("feedback_inline_dislike_success").incr() + true + } else { + scopedStats.counter("feedback_inline_dislike_filtered").incr() + false + } + } + + private def isPromptDislikeOutsideFatiguePeriod( + feedbacks: Seq[FeedbackModel], + filteredHistory: History, + feedbackFilters: Seq[Seq[FeedbackModel] => Seq[FeedbackModel]], + inlineFeedbackFatigueParam: ContinuousFunctionParam, + stats: StatsReceiver + ): Boolean = { + val scopedStats = stats.scope("prompt_dislike_fatigue") + + val promptDislikeFeedback = feedbacks + .filter(feedback => feedback.feedbackTypeEnum == FeedbackTypeEnum.PromptIrrelevant) + val hydratedPromptDislikeFeedback = FeedbackModelHydrator.HydrateNotification( + promptDislikeFeedback, + filteredHistory.history.toSeq.map(_._2)) + + if (isOutsideFatiguePeriod( + filteredHistory, + Seq(), + feedbackFilters.foldLeft(hydratedPromptDislikeFeedback)((feedbacks, feedbackFilter) => + feedbackFilter(feedbacks)), + inlineFeedbackFatigueParam, + scopedStats + )) { + scopedStats.counter("feedback_prompt_dislike_success").incr() + true + } else { + scopedStats.counter("feedback_prompt_dislike_filtered").incr() + false + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.scala new file mode 100644 index 0000000000..862541b633 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/ContinuousFunction.scala @@ -0,0 +1,148 @@ +package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue + +import com.twitter.finagle.stats.StatsReceiver + +case class ContinuousFunctionParam( + knobs: Seq[Double], + knobValues: Seq[Double], + powers: Seq[Double], + weight: Double, + defaultValue: Double) { + + def validateParams(): Boolean = { + knobs.size > 0 && knobs.size - 1 == powers.size && knobs.size == knobValues.size + } +} + +object ContinuousFunction { + + /** + * Evalutate the value for function f(x) = w(x - b)^power + * where w and b are decided by the start, startVal, end, endVal + * such that + * w(start - b) ^ power = startVal + * w(end - b) ^ power = endVal + * + * @param value the value at which we will evaluate the param + * @return weight * f(value) + */ + def evaluateFn( + value: Double, + start: Double, + startVal: Double, + end: Double, + endVal: Double, + power: Double, + weight: Double + ): Double = { + val b = + (math.pow(startVal / endVal, 1 / power) * end - start) / (math.pow( + startVal / endVal, + 1 / power) - 1) + val w = startVal / math.pow(start - b, power) + weight * w * math.pow(value - b, power) + } + + /** + * Evaluate value for function f(x), and return weight * f(x) + * + * f(x) is a piecewise function + * f(x) = w_i * (x - b_i)^powers[i] for knobs[i] <= x < knobs[i+1] + * such that + * w(knobs[i] - b) ^ power = knobVals[i] + * w(knobs[i+1] - b) ^ power = knobVals[i+1] + * + * @return Evaluate value for weight * f(x), for the function described above. If the any of the input is invalid, returns defaultVal + */ + def safeEvaluateFn( + value: Double, + knobs: Seq[Double], + knobVals: Seq[Double], + powers: Seq[Double], + weight: Double, + defaultVal: Double, + statsReceiver: StatsReceiver + ): Double = { + val totalStats = statsReceiver.counter("safe_evalfn_total") + val validStats = + statsReceiver.counter("safe_evalfn_valid") + val validEndCaseStats = + statsReceiver.counter("safe_evalfn_valid_endcase") + val invalidStats = statsReceiver.counter("safe_evalfn_invalid") + + totalStats.incr() + if (knobs.size <= 0 || knobs.size - 1 != powers.size || knobs.size != knobVals.size) { + invalidStats.incr() + defaultVal + } else { + val endIndex = knobs.indexWhere(knob => knob > value) + validStats.incr() + endIndex match { + case -1 => { + validEndCaseStats.incr() + knobVals(knobVals.size - 1) * weight + } + case 0 => { + validEndCaseStats.incr() + knobVals(0) * weight + } + case _ => { + val startIndex = endIndex - 1 + evaluateFn( + value, + knobs(startIndex), + knobVals(startIndex), + knobs(endIndex), + knobVals(endIndex), + powers(startIndex), + weight) + } + } + } + } + + def safeEvaluateFn( + value: Double, + fnParams: ContinuousFunctionParam, + statsReceiver: StatsReceiver + ): Double = { + val totalStats = statsReceiver.counter("safe_evalfn_total") + val validStats = + statsReceiver.counter("safe_evalfn_valid") + val validEndCaseStats = + statsReceiver.counter("safe_evalfn_valid_endcase") + val invalidStats = statsReceiver.counter("safe_evalfn_invalid") + + totalStats.incr() + + if (fnParams.validateParams()) { + val endIndex = fnParams.knobs.indexWhere(knob => knob > value) + validStats.incr() + endIndex match { + case -1 => { + validEndCaseStats.incr() + fnParams.knobValues(fnParams.knobValues.size - 1) * fnParams.weight + } + case 0 => { + validEndCaseStats.incr() + fnParams.knobValues(0) * fnParams.weight + } + case _ => { + val startIndex = endIndex - 1 + evaluateFn( + value, + fnParams.knobs(startIndex), + fnParams.knobValues(startIndex), + fnParams.knobs(endIndex), + fnParams.knobValues(endIndex), + fnParams.powers(startIndex), + fnParams.weight + ) + } + } + } else { + invalidStats.incr() + fnParams.defaultValue + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.scala new file mode 100644 index 0000000000..6548899011 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/FeedbackModel.scala @@ -0,0 +1,136 @@ +package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue + +import com.twitter.notificationservice.thriftscala.GenericType +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue +import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails +import com.twitter.notificationservice.feedback.thriftscala.FeedbackMetadata +import com.twitter.notificationservice.feedback.thriftscala.InlineFeedback +import com.twitter.notificationservice.feedback.thriftscala.FeedbackValue +import com.twitter.notificationservice.feedback.thriftscala.YesOrNoAnswer + +object FeedbackTypeEnum extends Enumeration { + val Unknown = Value + val CaretDislike = Value + val InlineDislike = Value + val InlineLike = Value + val InlineRevertedLike = Value + val InlineRevertedDislike = Value + val PromptRelevant = Value + val PromptIrrelevant = Value + val InlineDismiss = Value + val InlineRevertedDismiss = Value + val InlineSeeLess = Value + val InlineRevertedSeeLess = Value + val InlineNotRelevant = Value + val InlineRevertedNotRelevant = Value + + def safeFindByName(name: String): Value = + values.find(_.toString.toLowerCase() == name.toLowerCase()).getOrElse(Unknown) +} + +trait FeedbackModel { + + def timestampMs: Long + + def feedbackTypeEnum: FeedbackTypeEnum.Value + + def notificationImpressionId: Option[String] + + def notification: Option[FrigateNotification] = None +} + +case class CaretFeedbackModel( + caretFeedbackDetails: CaretFeedbackDetails, + notificationOpt: Option[FrigateNotification] = None) + extends FeedbackModel { + + override def timestampMs: Long = caretFeedbackDetails.eventTimestamp + + override def feedbackTypeEnum: FeedbackTypeEnum.Value = FeedbackTypeEnum.CaretDislike + + override def notificationImpressionId: Option[String] = caretFeedbackDetails.impressionId + + override def notification: Option[FrigateNotification] = notificationOpt + + def notificationGenericType: Option[GenericType] = { + caretFeedbackDetails.genericNotificationMetadata match { + case Some(genericNotificationMetadata) => + Some(genericNotificationMetadata.genericType) + case None => None + } + } +} + +case class InlineFeedbackModel( + feedback: FeedbackPromptValue, + notificationOpt: Option[FrigateNotification] = None) + extends FeedbackModel { + + override def timestampMs: Long = feedback.createdAt.inMilliseconds + + override def feedbackTypeEnum: FeedbackTypeEnum.Value = { + feedback.feedbackValue match { + case FeedbackValue( + _, + _, + _, + Some(FeedbackMetadata.InlineFeedback(InlineFeedback(Some(answer))))) => + FeedbackTypeEnum.safeFindByName("inline" + answer) + case _ => FeedbackTypeEnum.Unknown + } + } + + override def notificationImpressionId: Option[String] = Some(feedback.feedbackValue.impressionId) + + override def notification: Option[FrigateNotification] = notificationOpt +} + +case class PromptFeedbackModel( + feedback: FeedbackPromptValue, + notificationOpt: Option[FrigateNotification] = None) + extends FeedbackModel { + + override def timestampMs: Long = feedback.createdAt.inMilliseconds + + override def feedbackTypeEnum: FeedbackTypeEnum.Value = { + feedback.feedbackValue match { + case FeedbackValue(_, _, _, Some(FeedbackMetadata.YesOrNoAnswer(answer))) => + answer match { + case YesOrNoAnswer.Yes => FeedbackTypeEnum.PromptRelevant + case YesOrNoAnswer.No => FeedbackTypeEnum.PromptIrrelevant + case _ => FeedbackTypeEnum.Unknown + } + case _ => FeedbackTypeEnum.Unknown + } + } + + override def notificationImpressionId: Option[String] = Some(feedback.feedbackValue.impressionId) + + override def notification: Option[FrigateNotification] = notificationOpt +} + +object FeedbackModelHydrator { + + def HydrateNotification( + feedbacks: Seq[FeedbackModel], + history: Seq[FrigateNotification] + ): Seq[FeedbackModel] = { + feedbacks.map { + case feedback @ (inlineFeedback: InlineFeedbackModel) => + inlineFeedback.copy(notificationOpt = history.find( + _.impressionId + .equals(feedback.notificationImpressionId))) + case feedback @ (caretFeedback: CaretFeedbackModel) => + caretFeedback.copy(notificationOpt = history.find( + _.impressionId + .equals(feedback.notificationImpressionId))) + case feedback @ (promptFeedback: PromptFeedbackModel) => + promptFeedback.copy(notificationOpt = history.find( + _.impressionId + .equals(feedback.notificationImpressionId))) + case feedback => feedback + } + + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.scala new file mode 100644 index 0000000000..0405436609 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/MagicFanoutNtabCaretFatiguePredicate.scala @@ -0,0 +1,28 @@ +package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.predicate.NamedPredicate + +object MagicFanoutNtabCaretFatiguePredicate { + val name = "MagicFanoutNtabCaretFatiguePredicateForCandidate" + + private val MomentsCategory = "Moments" + private val MomentsViaMagicRecsCategory = "MomentsViaMagicRecs" + + def apply()(implicit globalStats: StatsReceiver): NamedPredicate[PushCandidate] = { + val scopedStats = globalStats.scope(name) + val genericTypeCategories = Seq(MomentsCategory, MomentsViaMagicRecsCategory) + val crts = RecTypes.magicFanoutEventTypes + RecTypeNtabCaretClickFatiguePredicate + .apply( + genericTypeCategories, + crts, + NtabCaretClickFatiguePredicateHelper.calculateFatiguePeriodMagicRecs, + useMostRecentDislikeTime = true, + name = name + ).withStats(scopedStats).withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.scala new file mode 100644 index 0000000000..376d9b11fb --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickCandidateFatiguePredicate.scala @@ -0,0 +1,87 @@ +package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.predicate.FatiguePredicate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.frigate.common.base.Candidate +import com.twitter.frigate.common.base.TargetInfo +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.common.base.{RecommendationType => BaseRecommendationType} +import com.twitter.frigate.common.predicate.CandidateWithRecommendationTypeAndTargetInfoWithCaretFeedbackHistory +import com.twitter.frigate.common.predicate.FrigateHistoryFatiguePredicate.TimeSeries +import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.predicate.CaretFeedbackHistoryFilter + +object NtabCaretClickContFnFatiguePredicate { + + private val MagicRecsCategory = "MagicRecs" + + def ntabCaretClickContFnFatiguePredicates( + filterHistory: TimeSeries => TimeSeries = + FatiguePredicate.recTypesOnlyFilter(RecTypes.sharedNTabCaretFatigueTypes), + filterCaretFeedbackHistory: Target => Seq[ + CaretFeedbackDetails + ] => Seq[CaretFeedbackDetails] = + CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(Seq(MagicRecsCategory)), + filterInlineFeedbackHistory: Seq[FeedbackModel] => Seq[FeedbackModel] = + NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(RecTypes.sharedNTabCaretFatigueTypes), + name: String = "NTabCaretClickFnCandidatePredicates" + )( + implicit globalStats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val scopedStats = globalStats.scope(name) + CRTBasedNtabCaretClickFatiguePredicates + .f1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ + Candidate with BaseRecommendationType with TargetInfo[ + Target + ] + ]( + filterHistory = filterHistory, + filterCaretFeedbackHistory = filterCaretFeedbackHistory, + filterInlineFeedbackHistory = filterInlineFeedbackHistory + ) + .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory + .withName("f1_triggered_fn_seelessoften_fatigue") + .andThen( + CRTBasedNtabCaretClickFatiguePredicates + .nonF1TriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ + Candidate with BaseRecommendationType with TargetInfo[ + Target + ] + ]( + filterHistory = filterHistory, + filterCaretFeedbackHistory = filterCaretFeedbackHistory, + filterInlineFeedbackHistory = filterInlineFeedbackHistory + ) + .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory) + .withName("nonf1_triggered_fn_seelessoften_fatigue") + .andThen( + CRTBasedNtabCaretClickFatiguePredicates + .tripHqTweetTriggeredCRTBasedNtabCaretClickFnFatiguePredicate[ + Candidate with BaseRecommendationType with TargetInfo[ + Target + ] + ]( + filterHistory = filterHistory, + filterCaretFeedbackHistory = filterCaretFeedbackHistory, + filterInlineFeedbackHistory = filterInlineFeedbackHistory + ) + .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory) + .withName("trip_hq_tweet_triggered_fn_seelessoften_fatigue") + .andThen( + CRTBasedNtabCaretClickFatiguePredicates + .genericCRTBasedNtabCaretClickFnFatiguePredicate[ + Candidate with BaseRecommendationType with TargetInfo[ + Target + ] + ]( + filterHistory = filterHistory, + filterCaretFeedbackHistory = filterCaretFeedbackHistory, + filterInlineFeedbackHistory = filterInlineFeedbackHistory) + .applyOnlyToCandidateWithRecommendationTypeAndTargetWithCaretFeedbackHistory + .withName("generic_fn_seelessoften_fatigue") + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.scala new file mode 100644 index 0000000000..579f4b25f5 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatiguePredicate.scala @@ -0,0 +1,47 @@ +package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +object NtabCaretClickFatiguePredicate { + val name = "NtabCaretClickFatiguePredicate" + + def isSpacesTypeAndTeamMember(candidate: PushCandidate): Future[Boolean] = { + candidate.target.isTeamMember.map { isTeamMember => + val isSpacesType = RecTypes.isRecommendedSpacesType(candidate.commonRecType) + isTeamMember && isSpacesType + } + } + + def apply()(implicit globalStats: StatsReceiver): NamedPredicate[PushCandidate] = { + val scopedStats = globalStats.scope(name) + val genericTypeCategories = Seq("MagicRecs") + val crts = RecTypes.sharedNTabCaretFatigueTypes + val recTypeNtabCaretClickFatiguePredicate = + RecTypeNtabCaretClickFatiguePredicate.apply( + genericTypeCategories, + crts, + NtabCaretClickFatiguePredicateHelper.calculateFatiguePeriodMagicRecs, + useMostRecentDislikeTime = false + ) + Predicate + .fromAsync { candidate: PushCandidate => + isSpacesTypeAndTeamMember(candidate).flatMap { isSpacesTypeAndTeamMember => + if (RecTypes.sharedNTabCaretFatigueTypes( + candidate.commonRecType) && !isSpacesTypeAndTeamMember) { + recTypeNtabCaretClickFatiguePredicate + .apply(Seq(candidate)).map(_.headOption.getOrElse(false)) + } else { + Future.True + } + } + } + .withStats(scopedStats) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.scala new file mode 100644 index 0000000000..cc2c0c0729 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/NtabCaretClickFatigueUtils.scala @@ -0,0 +1,108 @@ +package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicateHelper +import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails +import com.twitter.util.Duration +import com.twitter.conversions.DurationOps._ +import scala.math.min +import com.twitter.util.Time +import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} + +object NtabCaretClickFatigueUtils { + + private def pushCapForFeedback( + feedbackDetails: Seq[CaretFeedbackDetails], + feedbacks: Seq[FeedbackModel], + param: ContinuousFunctionParam, + statsReceiver: StatsReceiver + ): Double = { + val stats = statsReceiver.scope("mr_seelessoften_contfn_pushcap") + val pushCapTotal = stats.counter("pushcap_total") + val pushCapInvalid = + stats.counter("pushcap_invalid") + + pushCapTotal.incr() + val timeSinceMostRecentDislikeMs = + NtabCaretClickFatiguePredicateHelper.getDurationSinceMostRecentDislike(feedbackDetails) + val mostRecentFeedbackTimestamp: Option[Long] = + feedbacks + .map { feedback => + feedback.timestampMs + }.reduceOption(_ max _) + val timeSinceMostRecentFeedback: Option[Duration] = + mostRecentFeedbackTimestamp.map(Time.now - Time.fromMilliseconds(_)) + + val nTabDislikePushCap = timeSinceMostRecentDislikeMs match { + case Some(lastDislikeTimeMs) => { + ContinuousFunction.safeEvaluateFn(lastDislikeTimeMs.inDays.toDouble, param, stats) + } + case _ => { + pushCapInvalid.incr() + param.defaultValue + } + } + val feedbackPushCap = timeSinceMostRecentFeedback match { + case Some(lastDislikeTimeVal) => { + ContinuousFunction.safeEvaluateFn(lastDislikeTimeVal.inDays.toDouble, param, stats) + } + case _ => { + pushCapInvalid.incr() + param.defaultValue + } + } + + min(nTabDislikePushCap, feedbackPushCap) + } + + def durationToFilterForFeedback( + feedbackDetails: Seq[CaretFeedbackDetails], + feedbacks: Seq[FeedbackModel], + param: ContinuousFunctionParam, + defaultPushCap: Double, + statsReceiver: StatsReceiver + ): Duration = { + val pushCap = min( + pushCapForFeedback(feedbackDetails, feedbacks, param, statsReceiver), + defaultPushCap + ) + if (pushCap <= 0) { + Duration.Top + } else { + 24.hours / pushCap + } + } + + def hasUserDislikeInLast90Days(feedbackDetails: Seq[CaretFeedbackDetails]): Boolean = { + val timeSinceMostRecentDislike = + NtabCaretClickFatiguePredicateHelper.getDurationSinceMostRecentDislike(feedbackDetails) + + timeSinceMostRecentDislike.exists(_ < 90.days) + } + + def feedbackModelFilterByCRT( + crts: Set[CRT] + ): Seq[FeedbackModel] => Seq[ + FeedbackModel + ] = { feedbacks => + feedbacks.filter { feedback => + feedback.notification match { + case Some(notification) => crts.contains(notification.commonRecommendationType) + case None => false + } + } + } + + def feedbackModelExcludeCRT( + crts: Set[CRT] + ): Seq[FeedbackModel] => Seq[ + FeedbackModel + ] = { feedbacks => + feedbacks.filter { feedback => + feedback.notification match { + case Some(notification) => !crts.contains(notification.commonRecommendationType) + case None => true + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.scala new file mode 100644 index 0000000000..d83650de0c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.scala @@ -0,0 +1,87 @@ +package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.predicate.FatiguePredicate +import com.twitter.frigate.pushservice.predicate.CaretFeedbackHistoryFilter +import com.twitter.frigate.pushservice.predicate.{ + TargetNtabCaretClickFatiguePredicate => CommonNtabCaretClickFatiguePredicate +} +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.thriftscala.NotificationDisplayLocation +import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails +import com.twitter.util.Duration +import com.twitter.util.Future + +object RecTypeNtabCaretClickFatiguePredicate { + val defaultName = "RecTypeNtabCaretClickFatiguePredicateForCandidate" + + private def candidateFatiguePredicate( + genericTypeCategories: Seq[String], + crts: Set[CRT] + )( + implicit stats: StatsReceiver + ): NamedPredicate[ + PushCandidate + ] = { + val name = "f1TriggeredCRTBasedFatiguePredciate" + val scopedStats = stats.scope(s"predicate_$name") + Predicate + .fromAsync { candidate: PushCandidate => + if (candidate.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) { + if (candidate.target.params(PushParams.EnableFatigueNtabCaretClickingParam)) { + NtabCaretClickContFnFatiguePredicate + .ntabCaretClickContFnFatiguePredicates( + filterHistory = FatiguePredicate.recTypesOnlyFilter(crts), + filterCaretFeedbackHistory = + CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(genericTypeCategories), + filterInlineFeedbackHistory = + NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(crts) + ).apply(Seq(candidate)) + .map(_.headOption.getOrElse(false)) + } else Future.True + } else { + Future.True + } + }.withStats(scopedStats) + .withName(name) + } + + def apply( + genericTypeCategories: Seq[String], + crts: Set[CRT], + calculateFatiguePeriod: Seq[CaretFeedbackDetails] => Duration, + useMostRecentDislikeTime: Boolean, + name: String = defaultName + )( + implicit globalStats: StatsReceiver + ): NamedPredicate[PushCandidate] = { + val scopedStats = globalStats.scope(name) + val commonNtabCaretClickFatiguePredicate = CommonNtabCaretClickFatiguePredicate( + filterCaretFeedbackHistory = + CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(genericTypeCategories), + filterHistory = FatiguePredicate.recTypesOnlyFilter(crts), + calculateFatiguePeriod = calculateFatiguePeriod, + useMostRecentDislikeTime = useMostRecentDislikeTime, + name = name + )(globalStats) + + Predicate + .fromAsync { candidate: PushCandidate => + if (candidate.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) { + if (candidate.target.params(PushParams.EnableFatigueNtabCaretClickingParam)) { + commonNtabCaretClickFatiguePredicate + .apply(Seq(candidate.target)) + .map(_.headOption.getOrElse(false)) + } else Future.True + } else { + Future.True + } + }.andThen(candidateFatiguePredicate(genericTypeCategories, crts)) + .withStats(scopedStats) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.scala new file mode 100644 index 0000000000..61c1f78cc4 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.scala @@ -0,0 +1,44 @@ +package com.twitter.frigate.pushservice + +import com.twitter.frigate.common.base.Candidate +import com.twitter.frigate.common.base.SocialGraphServiceRelationshipMap +import com.twitter.frigate.common.base.TweetAuthor +import com.twitter.frigate.common.rec_types.RecTypes.isInNetworkTweetType +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.predicate.Predicate + +package object predicate { + implicit class CandidatesWithAuthorFollowPredicates( + predicate: Predicate[ + PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap + ]) { + def applyOnlyToAuthorBeingFollowPredicates: Predicate[Candidate] = + predicate.optionalOn[Candidate]( + { + case candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap + if isInNetworkTweetType(candidate.commonRecType) => + Some(candidate) + case _ => + None + }, + missingResult = true + ) + } + + implicit class TweetCandidateWithTweetAuthor( + predicate: Predicate[ + PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap + ]) { + def applyOnlyToBasicTweetPredicates: Predicate[Candidate] = + predicate.optionalOn[Candidate]( + { + case candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap + if isInNetworkTweetType(candidate.commonRecType) => + Some(candidate) + case _ => + None + }, + missingResult = true + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.scala new file mode 100644 index 0000000000..d7f38349b7 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.scala @@ -0,0 +1,27 @@ +package com.twitter.frigate.pushservice.predicate.quality_model_predicate + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.util.Future + +object ExplicitOONCFilterPredicate extends QualityPredicateBase { + override lazy val name = "open_or_ntab_click_explicit_threshold" + + override lazy val thresholdExtractor = (t: Target) => + Future.value(t.params(PushFeatureSwitchParams.QualityPredicateExplicitThresholdParam)) + + override def scoreExtractor = (candidate: PushCandidate) => + candidate.mrWeightedOpenOrNtabClickRankingProbability +} + +object WeightedOpenOrNtabClickQualityPredicate extends QualityPredicateBase { + override lazy val name = "weighted_open_or_ntab_click_model" + + override lazy val thresholdExtractor = (t: Target) => { + Future.value(0.0) + } + + override def scoreExtractor = + (candidate: PushCandidate) => candidate.mrWeightedOpenOrNtabClickFilteringProbability +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.scala new file mode 100644 index 0000000000..d22f8c68f7 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.scala @@ -0,0 +1,165 @@ +package com.twitter.frigate.pushservice.predicate.quality_model_predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.model.PushTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.target.TargetScoringDetails +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +object PDauCohort extends Enumeration { + type PDauCohort = Value + + val cohort1 = Value + val cohort2 = Value + val cohort3 = Value + val cohort4 = Value + val cohort5 = Value + val cohort6 = Value +} + +object PDauCohortUtil { + + case class DauThreshold( + threshold1: Double, + threshold2: Double, + threshold3: Double, + threshold4: Double, + threshold5: Double) + + val defaultDAUProb = 0.0 + + val dauProbThresholds = DauThreshold( + threshold1 = 0.05, + threshold2 = 0.14, + threshold3 = 0.33, + threshold4 = 0.7, + threshold5 = 0.959 + ) + + val finerThresholdMap = + Map( + PDauCohort.cohort2 -> List(0.05, 0.0539, 0.0563, 0.0600, 0.0681, 0.0733, 0.0800, 0.0849, + 0.0912, 0.0975, 0.1032, 0.1092, 0.1134, 0.1191, 0.1252, 0.1324, 0.14), + PDauCohort.cohort3 -> List(0.14, 0.1489, 0.1544, 0.1625, 0.1704, 0.1797, 0.1905, 0.2001, + 0.2120, 0.2248, 0.2363, 0.2500, 0.2650, 0.2801, 0.2958, 0.3119, 0.33), + PDauCohort.cohort4 -> List(0.33, 0.3484, 0.3686, 0.3893, 0.4126, 0.4350, 0.4603, 0.4856, + 0.5092, 0.5348, 0.5602, 0.5850, 0.6087, 0.6319, 0.6548, 0.6779, 0.7), + PDauCohort.cohort5 -> List(0.7, 0.7295, 0.7581, 0.7831, 0.8049, 0.8251, 0.8444, 0.8612, + 0.8786, 0.8936, 0.9043, 0.9175, 0.9290, 0.9383, 0.9498, 0.9587, 0.959) + ) + + def getBucket(targetUser: PushTypes.Target, doImpression: Boolean) = { + implicit val stats = targetUser.stats.scope("PDauCohortUtil") + if (doImpression) targetUser.getBucket _ else targetUser.getBucketWithoutImpression _ + } + + def threshold1(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold1 + + def threshold2(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold2 + + def threshold3(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold3 + + def threshold4(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold4 + + def threshold5(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold5 + + def thresholdForCohort(targetUser: PushTypes.Target, dauCohort: Int): Double = { + if (dauCohort == 0) 0.0 + else if (dauCohort == 1) threshold1(targetUser) + else if (dauCohort == 2) threshold2(targetUser) + else if (dauCohort == 3) threshold3(targetUser) + else if (dauCohort == 4) threshold4(targetUser) + else if (dauCohort == 5) threshold5(targetUser) + else 1.0 + } + + def getPDauCohort(dauProbability: Double, thresholds: DauThreshold): PDauCohort.Value = { + dauProbability match { + case dauProb if dauProb >= 0.0 && dauProb < thresholds.threshold1 => PDauCohort.cohort1 + case dauProb if dauProb >= thresholds.threshold1 && dauProb < thresholds.threshold2 => + PDauCohort.cohort2 + case dauProb if dauProb >= thresholds.threshold2 && dauProb < thresholds.threshold3 => + PDauCohort.cohort3 + case dauProb if dauProb >= thresholds.threshold3 && dauProb < thresholds.threshold4 => + PDauCohort.cohort4 + case dauProb if dauProb >= thresholds.threshold4 && dauProb < thresholds.threshold5 => + PDauCohort.cohort5 + case dauProb if dauProb >= thresholds.threshold5 && dauProb <= 1.0 => PDauCohort.cohort6 + } + } + + def getDauProb(target: TargetScoringDetails): Future[Double] = { + target.dauProbability.map { dauProb => + dauProb.map(_.probability).getOrElse(defaultDAUProb) + } + } + + def getPDauCohort(target: TargetScoringDetails): Future[PDauCohort.Value] = { + getDauProb(target).map { getPDauCohort(_, dauProbThresholds) } + } + + def getPDauCohortWithPDau(target: TargetScoringDetails): Future[(PDauCohort.Value, Double)] = { + getDauProb(target).map { prob => + (getPDauCohort(prob, dauProbThresholds), prob) + } + } + + def updateStats( + target: PushTypes.Target, + modelName: String, + predicateResult: Boolean + )( + implicit statsReceiver: StatsReceiver + ): Unit = { + val dauCohortOp = getPDauCohort(target) + dauCohortOp.map { dauCohort => + val cohortStats = statsReceiver.scope(modelName).scope(dauCohort.toString) + cohortStats.counter(s"filter_$predicateResult").incr() + } + if (target.isNewSignup) { + val newUserModelStats = statsReceiver.scope(modelName) + newUserModelStats.counter(s"new_user_filter_$predicateResult").incr() + } + } +} + +trait QualityPredicateBase { + def name: String + def thresholdExtractor: Target => Future[Double] + def scoreExtractor: PushCandidate => Future[Option[Double]] + def isPredicateEnabled: PushCandidate => Future[Boolean] = _ => Future.True + def comparator: (Double, Double) => Boolean = + (score: Double, threshold: Double) => score >= threshold + def updateCustomStats( + candidate: PushCandidate, + score: Double, + threshold: Double, + result: Boolean + )( + implicit statsReceiver: StatsReceiver + ): Unit = {} + + def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { + Predicate + .fromAsync { candidate: PushCandidate => + isPredicateEnabled(candidate).flatMap { + case true => + scoreExtractor(candidate).flatMap { scoreOpt => + thresholdExtractor(candidate.target).map { threshold => + val score = scoreOpt.getOrElse(0.0) + val result = comparator(score, threshold) + PDauCohortUtil.updateStats(candidate.target, name, result) + updateCustomStats(candidate, score, threshold, result) + result + } + } + case _ => Future.True + } + } + .withStats(statsReceiver.scope(s"predicate_$name")) + .withName(name) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.scala new file mode 100644 index 0000000000..9ac360df0b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.scala @@ -0,0 +1,21 @@ +package com.twitter.frigate.pushservice.predicate.quality_model_predicate + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.QualityPredicateEnum +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.hermit.predicate.NamedPredicate + +object QualityPredicateMap { + + def apply( + )( + implicit statsReceiver: StatsReceiver + ): Map[QualityPredicateEnum.Value, NamedPredicate[PushCandidate]] = { + Map( + QualityPredicateEnum.WeightedOpenOrNtabClick -> WeightedOpenOrNtabClickQualityPredicate(), + QualityPredicateEnum.ExplicitOpenOrNtabClickFilter -> ExplicitOONCFilterPredicate(), + QualityPredicateEnum.AlwaysTrue -> PredicatesForCandidate.alwaysTruePushCandidatePredicate, + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.scala new file mode 100644 index 0000000000..de95c0695c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.scala @@ -0,0 +1,54 @@ +package com.twitter.frigate.pushservice.rank + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.thriftscala.CommonRecommendationType + +/** + * This Ranker re-ranks MR candidates, boosting input CRTs. + * Relative ranking between input CRTs and rest of the candidates doesn't change + * + * Ex: T: Tweet candidate, F: input CRT candidatess + * + * T3, F2, T1, T2, F1 => F2, F1, T3, T1, T2 + */ +case class CRTBoostRanker(statsReceiver: StatsReceiver) { + + private val recsToBoostStat = statsReceiver.stat("recs_to_boost") + private val otherRecsStat = statsReceiver.stat("other_recs") + + private def boostCrtToTop( + inputCandidates: Seq[CandidateDetails[PushCandidate]], + crtToBoost: CommonRecommendationType + ): Seq[CandidateDetails[PushCandidate]] = { + val (upRankedCandidates, otherCandidates) = + inputCandidates.partition(_.candidate.commonRecType == crtToBoost) + recsToBoostStat.add(upRankedCandidates.size) + otherRecsStat.add(otherCandidates.size) + upRankedCandidates ++ otherCandidates + } + + final def boostCrtsToTop( + inputCandidates: Seq[CandidateDetails[PushCandidate]], + crtsToBoost: Seq[CommonRecommendationType] + ): Seq[CandidateDetails[PushCandidate]] = { + crtsToBoost.headOption match { + case Some(crt) => + val upRankedCandidates = boostCrtToTop(inputCandidates, crt) + boostCrtsToTop(upRankedCandidates, crtsToBoost.tail) + case None => inputCandidates + } + } + + final def boostCrtsToTopStableOrder( + inputCandidates: Seq[CandidateDetails[PushCandidate]], + crtsToBoost: Seq[CommonRecommendationType] + ): Seq[CandidateDetails[PushCandidate]] = { + val crtsToBoostSet = crtsToBoost.toSet + val (upRankedCandidates, otherCandidates) = inputCandidates.partition(candidateDetail => + crtsToBoostSet.contains(candidateDetail.candidate.commonRecType)) + + upRankedCandidates ++ otherCandidates + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.scala new file mode 100644 index 0000000000..4a8d74504b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.scala @@ -0,0 +1,45 @@ +package com.twitter.frigate.pushservice.rank + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.thriftscala.CommonRecommendationType + +/** + * This Ranker re-ranks MR candidates, down ranks input CRTs. + * Relative ranking between input CRTs and rest of the candidates doesn't change + * + * Ex: T: Tweet candidate, F: input CRT candidates + * + * T3, F2, T1, T2, F1 => T3, T1, T2, F2, F1 + */ +case class CRTDownRanker(statsReceiver: StatsReceiver) { + + private val recsToDownRankStat = statsReceiver.stat("recs_to_down_rank") + private val otherRecsStat = statsReceiver.stat("other_recs") + private val downRankerRequests = statsReceiver.counter("down_ranker_requests") + + private def downRank( + inputCandidates: Seq[CandidateDetails[PushCandidate]], + crtToDownRank: CommonRecommendationType + ): Seq[CandidateDetails[PushCandidate]] = { + downRankerRequests.incr() + val (downRankedCandidates, otherCandidates) = + inputCandidates.partition(_.candidate.commonRecType == crtToDownRank) + recsToDownRankStat.add(downRankedCandidates.size) + otherRecsStat.add(otherCandidates.size) + otherCandidates ++ downRankedCandidates + } + + final def downRank( + inputCandidates: Seq[CandidateDetails[PushCandidate]], + crtsToDownRank: Seq[CommonRecommendationType] + ): Seq[CandidateDetails[PushCandidate]] = { + crtsToDownRank.headOption match { + case Some(crt) => + val downRankedCandidates = downRank(inputCandidates, crt) + downRank(downRankedCandidates, crtsToDownRank.tail) + case None => inputCandidates + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.scala new file mode 100644 index 0000000000..5ab0c240a1 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.scala @@ -0,0 +1,45 @@ +package com.twitter.frigate.pushservice.rank + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +class LoggedOutRanker(tweetyPieStore: ReadableStore[Long, TweetyPieResult], stats: StatsReceiver) { + private val statsReceiver = stats.scope(this.getClass.getSimpleName) + private val rankedCandidates = statsReceiver.counter("ranked_candidates_count") + + def rank( + candidates: Seq[CandidateDetails[PushCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + val tweetIds = candidates.map { cand => cand.candidate.asInstanceOf[TweetCandidate].tweetId } + val results = tweetyPieStore.multiGet(tweetIds.toSet).values.toSeq + val futureOfResults = Future.traverseSequentially(results)(r => r) + val tweetsFut = futureOfResults.map { tweetyPieResults => + tweetyPieResults.map(_.map(_.tweet)) + } + val sortedTweetsFuture = tweetsFut.map { tweets => + tweets + .map { tweet => + if (tweet.isDefined && tweet.get.counts.isDefined) { + tweet.get.id -> tweet.get.counts.get.favoriteCount.getOrElse(0L) + } else { + 0 -> 0L + } + }.sortBy(_._2)(Ordering[Long].reverse) + } + val finalCandidates = sortedTweetsFuture.map { sortedTweets => + sortedTweets + .map { tweet => + candidates.find(_.candidate.asInstanceOf[TweetCandidate].tweetId == tweet._1).orNull + }.filter { cand => cand != null } + } + finalCandidates.map { fc => + rankedCandidates.incr(fc.size) + } + finalCandidates + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.scala new file mode 100644 index 0000000000..372888a660 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.scala @@ -0,0 +1,204 @@ +package com.twitter.frigate.pushservice.rank + +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.util.Future + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.params.MrQualityUprankingPartialTypeEnum +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.params.PushConstants.OoncQualityCombinedScore + +object ModelBasedRanker { + + def rankBySpecifiedScore( + candidatesDetails: Seq[CandidateDetails[PushCandidate]], + scoreExtractor: PushCandidate => Future[Option[Double]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + + val scoredCandidatesFutures = candidatesDetails.map { cand => + scoreExtractor(cand.candidate).map { scoreOp => (cand, scoreOp.getOrElse(0.0)) } + } + + Future.collect(scoredCandidatesFutures).map { scores => + val sorted = scores.sortBy { candidateDetails => -1 * candidateDetails._2 } + sorted.map(_._1) + } + } + + def populatePredictionScoreStats( + candidatesDetails: Seq[CandidateDetails[PushCandidate]], + scoreExtractor: PushCandidate => Future[Option[Double]], + predictionScoreStats: StatsReceiver + ): Unit = { + val scoreScaleFactorForStat = 10000 + val statName = "prediction_scores" + candidatesDetails.map { + case CandidateDetails(candidate, source) => + val crt = candidate.commonRecType + scoreExtractor(candidate).map { scoreOp => + val scaledScore = (scoreOp.getOrElse(0.0) * scoreScaleFactorForStat).toFloat + predictionScoreStats.scope("all_candidates").stat(statName).add(scaledScore) + predictionScoreStats.scope(crt.toString()).stat(statName).add(scaledScore) + } + } + } + + def populateMrWeightedOpenOrNtabClickScoreStats( + candidatesDetails: Seq[CandidateDetails[PushCandidate]], + predictionScoreStats: StatsReceiver + ): Unit = { + populatePredictionScoreStats( + candidatesDetails, + candidate => candidate.mrWeightedOpenOrNtabClickRankingProbability, + predictionScoreStats + ) + } + + def populateMrQualityUprankingScoreStats( + candidatesDetails: Seq[CandidateDetails[PushCandidate]], + predictionScoreStats: StatsReceiver + ): Unit = { + populatePredictionScoreStats( + candidatesDetails, + candidate => candidate.mrQualityUprankingProbability, + predictionScoreStats + ) + } + + def rankByMrWeightedOpenOrNtabClickScore( + candidatesDetails: Seq[CandidateDetails[PushCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + + rankBySpecifiedScore( + candidatesDetails, + candidate => candidate.mrWeightedOpenOrNtabClickRankingProbability + ) + } + + def transformSigmoid( + score: Double, + weight: Double = 1.0, + bias: Double = 0.0 + ): Double = { + val base = -1.0 * (weight * score + bias) + val cappedBase = math.max(math.min(base, 100.0), -100.0) + 1.0 / (1.0 + math.exp(cappedBase)) + } + + def transformLinear( + score: Double, + bar: Double = 1.0 + ): Double = { + val positiveBar = math.abs(bar) + val cappedScore = math.max(math.min(score, positiveBar), -1.0 * positiveBar) + cappedScore / positiveBar + } + + def transformIdentity( + score: Double + ): Double = score + + def rankByQualityOoncCombinedScore( + candidatesDetails: Seq[CandidateDetails[PushCandidate]], + qualityScoreTransform: Double => Double, + qualityScoreBoost: Double = 1.0 + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + + rankBySpecifiedScore( + candidatesDetails, + candidate => { + val ooncScoreFutOpt: Future[Option[Double]] = + candidate.mrWeightedOpenOrNtabClickRankingProbability + val qualityScoreFutOpt: Future[Option[Double]] = + candidate.mrQualityUprankingProbability + Future + .join( + ooncScoreFutOpt, + qualityScoreFutOpt + ).map { + case (Some(ooncScore), Some(qualityScore)) => + val transformedQualityScore = qualityScoreTransform(qualityScore) + val combinedScore = ooncScore * (1.0 + qualityScoreBoost * transformedQualityScore) + candidate + .cacheExternalScore(OoncQualityCombinedScore, Future.value(Some(combinedScore))) + Some(combinedScore) + case _ => None + } + } + ) + } + + def rerankByProducerQualityOoncCombinedScore( + candidateDetails: Seq[CandidateDetails[PushCandidate]] + )( + implicit stat: StatsReceiver + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + val scopedStat = stat.scope("producer_quality_reranking") + val oonCandidates = candidateDetails.filter { + case CandidateDetails(pushCandidate: PushCandidate, _) => + tweetCandidateSelector(pushCandidate, MrQualityUprankingPartialTypeEnum.Oon) + } + + val rankedOonCandidatesFut = rankBySpecifiedScore( + oonCandidates, + candidate => { + val baseScoreFutureOpt: Future[Option[Double]] = { + val qualityCombinedScoreFutureOpt = + candidate.getExternalCachedScoreByName(OoncQualityCombinedScore) + val ooncScoreFutureOpt = candidate.mrWeightedOpenOrNtabClickRankingProbability + Future.join(qualityCombinedScoreFutureOpt, ooncScoreFutureOpt).map { + case (Some(qualityCombinedScore), _) => + scopedStat.counter("quality_combined_score").incr() + Some(qualityCombinedScore) + case (_, ooncScoreOpt) => + scopedStat.counter("oonc_score").incr() + ooncScoreOpt + } + } + baseScoreFutureOpt.map { + case Some(baseScore) => + val boostRatio = candidate.mrProducerQualityUprankingBoost.getOrElse(1.0) + if (boostRatio > 1.0) scopedStat.counter("author_uprank").incr() + else if (boostRatio < 1.0) scopedStat.counter("author_downrank").incr() + else scopedStat.counter("author_noboost").incr() + Some(baseScore * boostRatio) + case _ => + scopedStat.counter("empty_score").incr() + None + } + } + ) + + rankedOonCandidatesFut.map { rankedOonCandidates => + val sortedOonCandidateIterator = rankedOonCandidates.toIterator + candidateDetails.map { ooncRankedCandidate => + val isOon = tweetCandidateSelector( + ooncRankedCandidate.candidate, + MrQualityUprankingPartialTypeEnum.Oon) + + if (sortedOonCandidateIterator.hasNext && isOon) + sortedOonCandidateIterator.next() + else ooncRankedCandidate + } + } + } + + def tweetCandidateSelector( + pushCandidate: PushCandidate, + selectedCandidateType: MrQualityUprankingPartialTypeEnum.Value + ): Boolean = { + pushCandidate match { + case candidate: PushCandidate with TweetCandidate => + selectedCandidateType match { + case MrQualityUprankingPartialTypeEnum.Oon => + val crt = candidate.commonRecType + RecTypes.isOutOfNetworkTweetRecType(crt) || RecTypes.outOfNetworkTopicTweetTypes + .contains(crt) + case _ => true + } + case _ => false + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.scala new file mode 100644 index 0000000000..26a3a4239b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.scala @@ -0,0 +1,31 @@ +package com.twitter.frigate.pushservice.rank + +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.Ranker +import com.twitter.util.Future + +trait PushserviceRanker[T, C] extends Ranker[T, C] { + + /** + * Initial Ranking of input candidates + */ + def initialRank(target: T, candidates: Seq[CandidateDetails[C]]): Future[Seq[CandidateDetails[C]]] + + /** + * Re-ranks input ranked candidates. Useful when a subset of candidates are ranked + * by a different logic, while preserving the initial ranking for the rest + */ + def reRank( + target: T, + rankedCandidates: Seq[CandidateDetails[C]] + ): Future[Seq[CandidateDetails[C]]] + + /** + * Final ranking that does Initial + Rerank + */ + override final def rank(target: T, candidates: Seq[CandidateDetails[C]]): ( + Future[Seq[CandidateDetails[C]]] + ) = { + initialRank(target, candidates).flatMap { rankedCandidates => reRank(target, rankedCandidates) } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.scala new file mode 100644 index 0000000000..3fdae08c37 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.scala @@ -0,0 +1,139 @@ +package com.twitter.frigate.pushservice.rank +import com.twitter.contentrecommender.thriftscala.LightRankingCandidate +import com.twitter.contentrecommender.thriftscala.LightRankingFeatureHydrationContext +import com.twitter.contentrecommender.thriftscala.MagicRecsFeatureHydrationContext +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.RandomRanker +import com.twitter.frigate.common.base.Ranker +import com.twitter.frigate.common.base.TweetAuthor +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.ml.featurestore.lib.UserId +import com.twitter.nrel.lightranker.MagicRecsServeDataRecordLightRanker +import com.twitter.util.Future + +class RFPHLightRanker( + lightRanker: MagicRecsServeDataRecordLightRanker, + stats: StatsReceiver) + extends Ranker[Target, PushCandidate] { + + private val statsReceiver = stats.scope(this.getClass.getSimpleName) + + private val lightRankerCandidateCounter = statsReceiver.counter("light_ranker_candidate_count") + private val lightRankerRequestCounter = statsReceiver.counter("light_ranker_request_count") + private val lightRankingStats: StatsReceiver = statsReceiver.scope("light_ranking") + private val restrictLightRankingCounter: Counter = + lightRankingStats.counter("restrict_light_ranking") + private val selectedLightRankerScribedTargetCandidateCountStats: Stat = + lightRankingStats.stat("selected_light_ranker_scribed_target_candidate_count") + private val selectedLightRankerScribedCandidatesStats: Stat = + lightRankingStats.stat("selected_light_ranker_scribed_candidates") + private val lightRankingRandomBaselineStats: StatsReceiver = + statsReceiver.scope("light_ranking_random_baseline") + + override def rank( + target: Target, + candidates: Seq[CandidateDetails[PushCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + val enableLightRanker = target.params(PushFeatureSwitchParams.EnableLightRankingParam) + val restrictLightRanker = target.params(PushParams.RestrictLightRankingParam) + val lightRankerSelectionThreshold = + target.params(PushFeatureSwitchParams.LightRankingNumberOfCandidatesParam) + val randomRanker = RandomRanker[Target, PushCandidate]()(lightRankingRandomBaselineStats) + + if (enableLightRanker && candidates.length > lightRankerSelectionThreshold && !target.scribeFeatureForRequestScribe) { + val (tweetCandidates, nonTweetCandidates) = + candidates.partition { + case CandidateDetails(pushCandidate: PushCandidate with TweetCandidate, source) => true + case _ => false + } + val lightRankerSelectedTweetCandidatesFut = { + if (restrictLightRanker) { + restrictLightRankingCounter.incr() + lightRankThenTake( + target, + tweetCandidates + .asInstanceOf[Seq[CandidateDetails[PushCandidate with TweetCandidate]]], + PushConstants.RestrictLightRankingCandidatesThreshold + ) + } else if (target.params(PushFeatureSwitchParams.EnableRandomBaselineLightRankingParam)) { + randomRanker.rank(target, tweetCandidates).map { randomLightRankerCands => + randomLightRankerCands.take(lightRankerSelectionThreshold) + } + } else { + lightRankThenTake( + target, + tweetCandidates + .asInstanceOf[Seq[CandidateDetails[PushCandidate with TweetCandidate]]], + lightRankerSelectionThreshold + ) + } + } + lightRankerSelectedTweetCandidatesFut.map { returnedTweetCandidates => + nonTweetCandidates ++ returnedTweetCandidates + } + } else if (target.scribeFeatureForRequestScribe) { + val downSampleRate: Double = + if (target.params(PushParams.DownSampleLightRankingScribeCandidatesParam)) + PushConstants.DownSampleLightRankingScribeCandidatesRate + else target.params(PushFeatureSwitchParams.LightRankingScribeCandidatesDownSamplingParam) + val selectedCandidateCounter: Int = math.ceil(candidates.size * downSampleRate).toInt + selectedLightRankerScribedTargetCandidateCountStats.add(selectedCandidateCounter.toFloat) + + randomRanker.rank(target, candidates).map { randomLightRankerCands => + val selectedCandidates = randomLightRankerCands.take(selectedCandidateCounter) + selectedLightRankerScribedCandidatesStats.add(selectedCandidates.size.toFloat) + selectedCandidates + } + } else Future.value(candidates) + } + + private def lightRankThenTake( + target: Target, + candidates: Seq[CandidateDetails[PushCandidate with TweetCandidate]], + numOfCandidates: Int + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + lightRankerCandidateCounter.incr(candidates.length) + lightRankerRequestCounter.incr() + val lightRankerCandidates: Seq[LightRankingCandidate] = candidates.map { + case CandidateDetails(tweetCandidate, _) => + val tweetAuthor = tweetCandidate match { + case t: TweetCandidate with TweetAuthor => t.authorId + case _ => None + } + val hydrationContext: LightRankingFeatureHydrationContext = + LightRankingFeatureHydrationContext.MagicRecsHydrationContext( + MagicRecsFeatureHydrationContext( + tweetAuthor = tweetAuthor, + pushString = tweetCandidate.getPushCopy.flatMap(_.pushStringGroup).map(_.toString)) + ) + LightRankingCandidate( + tweetId = tweetCandidate.tweetId, + hydrationContext = Some(hydrationContext) + ) + } + val modelName = target.params(PushFeatureSwitchParams.LightRankingModelTypeParam) + val lightRankedCandidatesFut = { + lightRanker + .rank(UserId(target.targetId), lightRankerCandidates, modelName) + } + + lightRankedCandidatesFut.map { lightRankedCandidates => + val lrScoreMap = lightRankedCandidates.map { lrCand => + lrCand.tweetId -> lrCand.score + }.toMap + val candScoreMap: Seq[Option[Double]] = candidates.map { candidateDetails => + lrScoreMap.get(candidateDetails.candidate.tweetId) + } + sortCandidatesByScore(candidates, candScoreMap) + .take(numOfCandidates) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.scala new file mode 100644 index 0000000000..83bdf3932d --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.scala @@ -0,0 +1,297 @@ +package com.twitter.frigate.pushservice.rank +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.Ranker +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.ml.HealthFeatureGetter +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.params.MrQualityUprankingPartialTypeEnum +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushMLModel +import com.twitter.frigate.pushservice.params.PushModelName +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil.updateMediaCategoryStats +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.util.Future +import com.twitter.frigate.pushservice.params.MrQualityUprankingTransformTypeEnum +import com.twitter.storehaus.ReadableStore +import com.twitter.frigate.thriftscala.UserMediaRepresentation +import com.twitter.hss.api.thriftscala.UserHealthSignalResponse + +class RFPHRanker( + randomRanker: Ranker[Target, PushCandidate], + weightedOpenOrNtabClickModelScorer: PushMLModelScorer, + subscriptionCreatorRanker: SubscriptionCreatorRanker, + userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse], + producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation], + stats: StatsReceiver) + extends PushserviceRanker[Target, PushCandidate] { + + private val statsReceiver = stats.scope(this.getClass.getSimpleName) + + private val boostCRTsRanker = CRTBoostRanker(statsReceiver.scope("boost_desired_crts")) + private val crtDownRanker = CRTDownRanker(statsReceiver.scope("down_rank_desired_crts")) + + private val crtsToDownRank = statsReceiver.stat("crts_to_downrank") + private val crtsToUprank = statsReceiver.stat("crts_to_uprank") + + private val randomRankingCounter = stats.counter("randomRanking") + private val mlRankingCounter = stats.counter("mlRanking") + private val disableAllRelevanceCounter = stats.counter("disableAllRelevance") + private val disableHeavyRankingCounter = stats.counter("disableHeavyRanking") + + private val heavyRankerCandidateCounter = stats.counter("heavy_ranker_candidate_count") + private val heavyRankerScoreStats = statsReceiver.scope("heavy_ranker_prediction_scores") + + private val producerUprankingCounter = statsReceiver.counter("producer_quality_upranking") + private val producerBoostedCounter = statsReceiver.counter("producer_boosted_candidates") + private val producerDownboostedCounter = statsReceiver.counter("producer_downboosted_candidates") + + override def initialRank( + target: Target, + candidates: Seq[CandidateDetails[PushCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + + heavyRankerCandidateCounter.incr(candidates.size) + + updateMediaCategoryStats(candidates)(stats) + target.targetUserState + .flatMap { targetUserState => + val useRandomRanking = target.skipMlRanker || target.params( + PushParams.UseRandomRankingParam + ) + + if (useRandomRanking) { + randomRankingCounter.incr() + randomRanker.rank(target, candidates) + } else if (target.params(PushParams.DisableAllRelevanceParam)) { + disableAllRelevanceCounter.incr() + Future.value(candidates) + } else if (target.params(PushParams.DisableHeavyRankingParam) || target.params( + PushFeatureSwitchParams.DisableHeavyRankingModelFSParam)) { + disableHeavyRankingCounter.incr() + Future.value(candidates) + } else { + mlRankingCounter.incr() + + val scoredCandidatesFut = scoring(target, candidates) + + target.rankingModelParam.map { rankingModelParam => + val modelName = PushModelName( + PushMLModel.WeightedOpenOrNtabClickProbability, + target.params(rankingModelParam)).toString + ModelBasedRanker.populateMrWeightedOpenOrNtabClickScoreStats( + candidates, + heavyRankerScoreStats.scope(modelName) + ) + } + + if (target.params( + PushFeatureSwitchParams.EnableQualityUprankingCrtScoreStatsForHeavyRankingParam)) { + val modelName = PushModelName( + PushMLModel.FilteringProbability, + target.params(PushFeatureSwitchParams.QualityUprankingModelTypeParam) + ).toString + ModelBasedRanker.populateMrQualityUprankingScoreStats( + candidates, + heavyRankerScoreStats.scope(modelName) + ) + } + + val ooncRankedCandidatesFut = + scoredCandidatesFut.flatMap(ModelBasedRanker.rankByMrWeightedOpenOrNtabClickScore) + + val qualityUprankedCandidatesFut = + if (target.params(PushFeatureSwitchParams.EnableQualityUprankingForHeavyRankingParam)) { + ooncRankedCandidatesFut.flatMap { ooncRankedCandidates => + val transformFunc: Double => Double = + target.params(PushFeatureSwitchParams.QualityUprankingTransformTypeParam) match { + case MrQualityUprankingTransformTypeEnum.Linear => + ModelBasedRanker.transformLinear( + _, + bar = target.params( + PushFeatureSwitchParams.QualityUprankingLinearBarForHeavyRankingParam)) + case MrQualityUprankingTransformTypeEnum.Sigmoid => + ModelBasedRanker.transformSigmoid( + _, + weight = target.params( + PushFeatureSwitchParams.QualityUprankingSigmoidWeightForHeavyRankingParam), + bias = target.params( + PushFeatureSwitchParams.QualityUprankingSigmoidBiasForHeavyRankingParam) + ) + case _ => ModelBasedRanker.transformIdentity + } + + ModelBasedRanker.rankByQualityOoncCombinedScore( + ooncRankedCandidates, + transformFunc, + target.params(PushFeatureSwitchParams.QualityUprankingBoostForHeavyRankingParam) + ) + } + } else ooncRankedCandidatesFut + + if (target.params( + PushFeatureSwitchParams.EnableProducersQualityBoostingForHeavyRankingParam)) { + producerUprankingCounter.incr() + qualityUprankedCandidatesFut.flatMap(cands => + ModelBasedRanker.rerankByProducerQualityOoncCombinedScore(cands)(statsReceiver)) + } else qualityUprankedCandidatesFut + } + } + } + + private def scoring( + target: Target, + candidates: Seq[CandidateDetails[PushCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + + val ooncScoredCandidatesFut = target.rankingModelParam.map { rankingModelParam => + weightedOpenOrNtabClickModelScorer.scoreByBatchPredictionForModelVersion( + target, + candidates, + rankingModelParam + ) + } + + val scoredCandidatesFut = { + if (target.params(PushFeatureSwitchParams.EnableQualityUprankingForHeavyRankingParam)) { + ooncScoredCandidatesFut.map { candidates => + weightedOpenOrNtabClickModelScorer.scoreByBatchPredictionForModelVersion( + target = target, + candidatesDetails = candidates, + modelVersionParam = PushFeatureSwitchParams.QualityUprankingModelTypeParam, + overridePushMLModelOpt = Some(PushMLModel.FilteringProbability) + ) + } + } else ooncScoredCandidatesFut + } + + scoredCandidatesFut.foreach { candidates => + val oonCandidates = candidates.filter { + case CandidateDetails(pushCandidate: PushCandidate, _) => + ModelBasedRanker.tweetCandidateSelector( + pushCandidate, + MrQualityUprankingPartialTypeEnum.Oon) + } + setProducerQuality( + target, + oonCandidates, + userHealthSignalStore, + producerMediaRepresentationStore) + } + } + + private def setProducerQuality( + target: Target, + candidates: Seq[CandidateDetails[PushCandidate]], + userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse], + producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] + ): Unit = { + lazy val boostRatio = + target.params(PushFeatureSwitchParams.QualityUprankingBoostForHighQualityProducersParam) + lazy val downboostRatio = + target.params(PushFeatureSwitchParams.QualityUprankingDownboostForLowQualityProducersParam) + candidates.foreach { + case CandidateDetails(pushCandidate, _) => + HealthFeatureGetter + .getFeatures(pushCandidate, producerMediaRepresentationStore, userHealthSignalStore).map { + featureMap => + val agathaNsfwScore = featureMap.numericFeatures.getOrElse("agathaNsfwScore", 0.5) + val textNsfwScore = featureMap.numericFeatures.getOrElse("textNsfwScore", 0.15) + val nudityRate = featureMap.numericFeatures.getOrElse("nudityRate", 0.0) + val activeFollowers = featureMap.numericFeatures.getOrElse("activeFollowers", 0.0) + val favorsRcvd28Days = featureMap.numericFeatures.getOrElse("favorsRcvd28Days", 0.0) + val tweets28Days = featureMap.numericFeatures.getOrElse("tweets28Days", 0.0) + val authorDislikeCount = featureMap.numericFeatures + .getOrElse("authorDislikeCount", 0.0) + val authorDislikeRate = featureMap.numericFeatures.getOrElse("authorDislikeRate", 0.0) + val authorReportRate = featureMap.numericFeatures.getOrElse("authorReportRate", 0.0) + val abuseStrikeTop2Percent = + featureMap.booleanFeatures.getOrElse("abuseStrikeTop2Percent", false) + val abuseStrikeTop1Percent = + featureMap.booleanFeatures.getOrElse("abuseStrikeTop1Percent", false) + val hasNsfwToken = featureMap.booleanFeatures.getOrElse("hasNsfwToken", false) + + if ((activeFollowers > 3000000) || + (activeFollowers > 1000000 && agathaNsfwScore < 0.7 && nudityRate < 0.01 && !hasNsfwToken && !abuseStrikeTop2Percent) || + (activeFollowers > 100000 && agathaNsfwScore < 0.7 && nudityRate < 0.01 && !hasNsfwToken && !abuseStrikeTop2Percent && + tweets28Days > 0 && favorsRcvd28Days / tweets28Days > 3000 && authorReportRate < 0.000001 && authorDislikeRate < 0.0005)) { + producerBoostedCounter.incr() + pushCandidate.setProducerQualityUprankingBoost(boostRatio) + } else if (activeFollowers < 5 || agathaNsfwScore > 0.9 || nudityRate > 0.03 || hasNsfwToken || abuseStrikeTop1Percent || + textNsfwScore > 0.4 || (authorDislikeRate > 0.005 && authorDislikeCount > 5) || + (tweets28Days > 56 && favorsRcvd28Days / tweets28Days < 100)) { + producerDownboostedCounter.incr() + pushCandidate.setProducerQualityUprankingBoost(downboostRatio) + } else pushCandidate.setProducerQualityUprankingBoost(1.0) + } + } + } + + private def rerankBySubscriptionCreatorRanker( + target: Target, + rankedCandidates: Future[Seq[CandidateDetails[PushCandidate]]], + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + if (target.params(PushFeatureSwitchParams.SoftRankCandidatesFromSubscriptionCreators)) { + val factor = target.params(PushFeatureSwitchParams.SoftRankFactorForSubscriptionCreators) + subscriptionCreatorRanker.boostByScoreFactor(rankedCandidates, factor) + } else + subscriptionCreatorRanker.boostSubscriptionCreator(rankedCandidates) + } + + override def reRank( + target: Target, + rankedCandidates: Seq[CandidateDetails[PushCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + val numberOfF1Candidates = + rankedCandidates.count(candidateDetails => + RecTypes.isF1Type(candidateDetails.candidate.commonRecType)) + lazy val threshold = + target.params(PushFeatureSwitchParams.NumberOfF1CandidatesThresholdForOONBackfill) + lazy val enableOONBackfillBasedOnF1 = + target.params(PushFeatureSwitchParams.EnableOONBackfillBasedOnF1Candidates) + + val f1BoostedCandidates = + if (enableOONBackfillBasedOnF1 && numberOfF1Candidates > threshold) { + boostCRTsRanker.boostCrtsToTopStableOrder( + rankedCandidates, + RecTypes.f1FirstDegreeTypes.toSeq) + } else rankedCandidates + + val topTweetsByGeoDownRankedCandidates = + if (target.params(PushFeatureSwitchParams.BackfillRankTopTweetsByGeoCandidates)) { + crtDownRanker.downRank( + f1BoostedCandidates, + Seq(CommonRecommendationType.GeoPopTweet) + ) + } else f1BoostedCandidates + + val reRankedCandidatesWithBoostedCrts = { + val listOfCrtsToUpRank = target + .params(PushFeatureSwitchParams.ListOfCrtsToUpRank) + .flatMap(CommonRecommendationType.valueOf) + crtsToUprank.add(listOfCrtsToUpRank.size) + boostCRTsRanker.boostCrtsToTop(topTweetsByGeoDownRankedCandidates, listOfCrtsToUpRank) + } + + val reRankedCandidatesWithDownRankedCrts = { + val listOfCrtsToDownRank = target + .params(PushFeatureSwitchParams.ListOfCrtsToDownRank) + .flatMap(CommonRecommendationType.valueOf) + crtsToDownRank.add(listOfCrtsToDownRank.size) + crtDownRanker.downRank(reRankedCandidatesWithBoostedCrts, listOfCrtsToDownRank) + } + + val rerankBySubscriptionCreatorFut = { + if (target.params(PushFeatureSwitchParams.BoostCandidatesFromSubscriptionCreators)) { + rerankBySubscriptionCreatorRanker( + target, + Future.value(reRankedCandidatesWithDownRankedCrts)) + } else Future.value(reRankedCandidatesWithDownRankedCrts) + } + + rerankBySubscriptionCreatorFut + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.scala new file mode 100644 index 0000000000..3a2bff9a57 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.scala @@ -0,0 +1,110 @@ +package com.twitter.frigate.pushservice.rank + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.TweetAuthor +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.storehaus.FutureOps +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +class SubscriptionCreatorRanker( + superFollowEligibilityUserStore: ReadableStore[Long, Boolean], + statsReceiver: StatsReceiver) { + + private val scopedStats = statsReceiver.scope("SubscriptionCreatorRanker") + private val boostStats = scopedStats.scope("boostSubscriptionCreator") + private val softUprankStats = scopedStats.scope("boostByScoreFactor") + private val boostTotalCandidates = boostStats.stat("total_input_candidates") + private val softRankTotalCandidates = softUprankStats.stat("total_input_candidates") + private val softRankNumCandidatesCreators = softUprankStats.counter("candidates_from_creators") + private val softRankNumCandidatesNonCreators = + softUprankStats.counter("candidates_not_from_creators") + private val boostNumCandidatesCreators = boostStats.counter("candidates_from_creators") + private val boostNumCandidatesNonCreators = + boostStats.counter("candidates_not_from_creators") + + def boostSubscriptionCreator( + inputCandidatesFut: Future[Seq[CandidateDetails[PushCandidate]]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + + inputCandidatesFut.flatMap { inputCandidates => + boostTotalCandidates.add(inputCandidates.size) + val tweetAuthorIds = inputCandidates.flatMap { + case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => + candidate.authorId + case _ => None + }.toSet + + FutureOps + .mapCollect(superFollowEligibilityUserStore.multiGet(tweetAuthorIds)) + .map { creatorAuthorMap => + val (upRankedCandidates, otherCandidates) = inputCandidates.partition { + case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => + candidate.authorId match { + case Some(authorId) => + creatorAuthorMap(authorId).getOrElse(false) + case _ => false + } + case _ => false + } + boostNumCandidatesCreators.incr(upRankedCandidates.size) + boostNumCandidatesNonCreators.incr(otherCandidates.size) + upRankedCandidates ++ otherCandidates + } + } + } + + def boostByScoreFactor( + inputCandidatesFut: Future[Seq[CandidateDetails[PushCandidate]]], + factor: Double = 1.0, + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + + inputCandidatesFut.flatMap { inputCandidates => + softRankTotalCandidates.add(inputCandidates.size) + val tweetAuthorIds = inputCandidates.flatMap { + case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => + candidate.authorId + case _ => None + }.toSet + + FutureOps + .mapCollect(superFollowEligibilityUserStore.multiGet(tweetAuthorIds)) + .flatMap { creatorAuthorMap => + val (upRankedCandidates, otherCandidates) = inputCandidates.partition { + case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => + candidate.authorId match { + case Some(authorId) => + creatorAuthorMap(authorId).getOrElse(false) + case _ => false + } + case _ => false + } + softRankNumCandidatesCreators.incr(upRankedCandidates.size) + softRankNumCandidatesNonCreators.incr(otherCandidates.size) + + ModelBasedRanker.rankBySpecifiedScore( + inputCandidates, + candidate => { + val isFromCreator = candidate match { + case candidate: TweetCandidate with TweetAuthor => + candidate.authorId match { + case Some(authorId) => + creatorAuthorMap(authorId).getOrElse(false) + case _ => false + } + case _ => false + } + candidate.mrWeightedOpenOrNtabClickRankingProbability.map { + case Some(score) => + if (isFromCreator) Some(score * factor) + else Some(score) + case _ => None + } + } + ) + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.scala new file mode 100644 index 0000000000..f626d5b089 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.scala @@ -0,0 +1,259 @@ +package com.twitter.frigate.pushservice.refresh_handler + +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.CandidateResult +import com.twitter.frigate.common.base.CandidateSource +import com.twitter.frigate.common.base.FetchRankFlowWithHydratedCandidates +import com.twitter.frigate.common.base.Invalid +import com.twitter.frigate.common.base.OK +import com.twitter.frigate.common.base.Response +import com.twitter.frigate.common.base.Result +import com.twitter.frigate.common.base.Stats.track +import com.twitter.frigate.common.base.Stats.trackSeq +import com.twitter.frigate.common.logger.MRLogger +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.adaptor.LoggedOutPushCandidateSourceGenerator +import com.twitter.frigate.pushservice.predicate.LoggedOutPreRankingPredicates +import com.twitter.frigate.pushservice.predicate.LoggedOutTargetPredicates +import com.twitter.frigate.pushservice.rank.LoggedOutRanker +import com.twitter.frigate.pushservice.take.LoggedOutRefreshForPushNotifier +import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler +import com.twitter.frigate.pushservice.target.LoggedOutPushTargetUserBuilder +import com.twitter.frigate.pushservice.thriftscala.LoggedOutRequest +import com.twitter.frigate.pushservice.thriftscala.LoggedOutResponse +import com.twitter.frigate.pushservice.thriftscala.PushContext +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.hermit.predicate.SequentialPredicate +import com.twitter.util.Future + +class LoggedOutRefreshForPushHandler( + val loPushTargetUserBuilder: LoggedOutPushTargetUserBuilder, + val loPushCandidateSourceGenerator: LoggedOutPushCandidateSourceGenerator, + candidateHydrator: PushCandidateHydrator, + val loRanker: LoggedOutRanker, + val loRfphNotifier: LoggedOutRefreshForPushNotifier, + loMrRequestScriberNode: String +)( + globalStats: StatsReceiver) + extends FetchRankFlowWithHydratedCandidates[Target, RawCandidate, PushCandidate] { + + val log = MRLogger("LORefreshForPushHandler") + implicit val statsReceiver: StatsReceiver = + globalStats.scope("LORefreshForPushHandler") + private val loggedOutBuildStats = statsReceiver.scope("logged_out_build_target") + private val loggedOutProcessStats = statsReceiver.scope("logged_out_process") + private val loggedOutNotifyStats = statsReceiver.scope("logged_out_notify") + private val loCandidateHydrationStats: StatsReceiver = + statsReceiver.scope("logged_out_candidate_hydration") + val mrLORequestCandidateScribeStats = + statsReceiver.scope("mr_logged_out_request_scribe_candidates") + + val mrRequestScribeHandler = + new MrRequestScribeHandler(loMrRequestScriberNode, statsReceiver.scope("lo_mr_request_scribe")) + val loMrRequestTargetScribeStats = statsReceiver.scope("lo_mr_request_scribe_target") + + lazy val loCandSourceEligibleCounter: Counter = + loCandidateStats.counter("logged_out_cand_source_eligible") + lazy val loCandSourceNotEligibleCounter: Counter = + loCandidateStats.counter("logged_out_cand_source_not_eligible") + lazy val allCandidatesCounter: Counter = statsReceiver.counter("all_logged_out_candidates") + val allCandidatesFilteredPreRank = filterStats.counter("all_logged_out_candidates_filtered") + + override def targetPredicates(target: Target): List[Predicate[Target]] = List( + LoggedOutTargetPredicates.targetFatiguePredicate(), + LoggedOutTargetPredicates.loggedOutRecsHoldbackPredicate() + ) + + override def isTargetValid(target: Target): Future[Result] = { + val resultFut = + if (target.skipFilters) { + Future.value(OK) + } else { + predicateSeq(target).track(Seq(target)).map { resultArr => + trackTargetPredStats(resultArr(0)) + } + } + track(targetStats)(resultFut) + } + + override def rank( + target: Target, + candidateDetails: Seq[CandidateDetails[PushCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + loRanker.rank(candidateDetails) + } + + override def validCandidates( + target: Target, + candidates: Seq[PushCandidate] + ): Future[Seq[Result]] = { + Future.value(candidates.map { c => OK }) + } + + override def desiredCandidateCount(target: Target): Int = 1 + + private val loggedOutPreRankingPredicates = + LoggedOutPreRankingPredicates(filterStats.scope("logged_out_predicates")) + + private val loggedOutPreRankingPredicateChain = + new SequentialPredicate[PushCandidate](loggedOutPreRankingPredicates) + + override def filter( + target: Target, + candidates: Seq[CandidateDetails[PushCandidate]] + ): Future[ + (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) + ] = { + val predicateChain = loggedOutPreRankingPredicateChain + predicateChain + .track(candidates.map(_.candidate)) + .map { results => + val resultForPreRankingFiltering = + results + .zip(candidates) + .foldLeft( + ( + Seq.empty[CandidateDetails[PushCandidate]], + Seq.empty[CandidateResult[PushCandidate, Result]] + ) + ) { + case ((goodCandidates, filteredCandidates), (result, candidateDetails)) => + result match { + case None => + (goodCandidates :+ candidateDetails, filteredCandidates) + + case Some(pred: NamedPredicate[_]) => + val r = Invalid(Some(pred.name)) + ( + goodCandidates, + filteredCandidates :+ CandidateResult[PushCandidate, Result]( + candidateDetails.candidate, + candidateDetails.source, + r + ) + ) + case Some(_) => + val r = Invalid(Some("Filtered by un-named predicate")) + ( + goodCandidates, + filteredCandidates :+ CandidateResult[PushCandidate, Result]( + candidateDetails.candidate, + candidateDetails.source, + r + ) + ) + } + } + resultForPreRankingFiltering match { + case (validCandidates, _) if validCandidates.isEmpty && candidates.nonEmpty => + allCandidatesFilteredPreRank.incr() + case _ => () + + } + resultForPreRankingFiltering + + } + + } + + override def candidateSources( + target: Target + ): Future[Seq[CandidateSource[Target, RawCandidate]]] = { + Future + .collect(loPushCandidateSourceGenerator.sources.map { cs => + cs.isCandidateSourceAvailable(target).map { isEligible => + if (isEligible) { + loCandSourceEligibleCounter.incr() + Some(cs) + } else { + loCandSourceNotEligibleCounter.incr() + None + } + } + }).map(_.flatten) + } + + override def process( + target: Target, + externalCandidates: Seq[RawCandidate] = Nil + ): Future[Response[PushCandidate, Result]] = { + isTargetValid(target).flatMap { + case OK => + for { + candidatesFromSources <- trackSeq(fetchStats)(fetchCandidates(target)) + externalCandidateDetails = externalCandidates.map( + CandidateDetails(_, "logged_out_refresh_for_push_handler_external_candidates")) + allCandidates = candidatesFromSources ++ externalCandidateDetails + hydratedCandidatesWithCopy <- + trackSeq(loCandidateHydrationStats)(hydrateCandidates(allCandidates)) + (candidates, preRankingFilteredCandidates) <- + track(filterStats)(filter(target, hydratedCandidatesWithCopy)) + rankedCandidates <- trackSeq(rankingStats)(rank(target, candidates)) + allTakeCandidateResults <- track(takeStats)( + take(target, rankedCandidates, desiredCandidateCount(target)) + ) + _ <- track(mrLORequestCandidateScribeStats)( + mrRequestScribeHandler.scribeForCandidateFiltering( + target, + hydratedCandidatesWithCopy, + preRankingFilteredCandidates, + rankedCandidates, + rankedCandidates, + rankedCandidates, + allTakeCandidateResults + )) + + } yield { + val takeCandidateResults = allTakeCandidateResults.filterNot { candResult => + candResult.result == MoreThanDesiredCandidates + } + val allCandidateResults = takeCandidateResults ++ preRankingFilteredCandidates + allCandidatesCounter.incr(allCandidateResults.size) + Response(OK, allCandidateResults) + } + + case result: Result => + for (_ <- track(loMrRequestTargetScribeStats)( + mrRequestScribeHandler.scribeForTargetFiltering(target, result))) yield { + Response(result, Nil) + } + } + } + + def buildTarget( + guestId: Long, + inputPushContext: Option[PushContext] + ): Future[Target] = + loPushTargetUserBuilder.buildTarget(guestId, inputPushContext) + + /** + * Hydrate candidate by querying downstream services + * + * @param candidates - candidates + * + * @return - hydrated candidates + */ + override def hydrateCandidates( + candidates: Seq[CandidateDetails[RawCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = candidateHydrator(candidates) + + override def batchForCandidatesCheck(target: Target): Int = 1 + + def refreshAndSend(request: LoggedOutRequest): Future[LoggedOutResponse] = { + for { + target <- track(loggedOutBuildStats)( + loPushTargetUserBuilder.buildTarget(request.guestId, request.context)) + response <- track(loggedOutProcessStats)(process(target, externalCandidates = Seq.empty)) + loggedOutRefreshResponse <- + track(loggedOutNotifyStats)(loRfphNotifier.checkResponseAndNotify(response)) + } yield { + loggedOutRefreshResponse + } + } + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.scala new file mode 100644 index 0000000000..b8bf675bd3 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.scala @@ -0,0 +1,239 @@ +package com.twitter.frigate.pushservice.refresh_handler + +import com.twitter.channels.common.thriftscala.ApiList +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.rec_types.RecTypes.isInNetworkTweetType +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.TrendTweetPushCandidate +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.refresh_handler.cross.CandidateCopyExpansion +import com.twitter.frigate.pushservice.util.CandidateHydrationUtil._ +import com.twitter.frigate.pushservice.util.MrUserStateUtil +import com.twitter.frigate.pushservice.util.RelationshipUtil +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +case class PushCandidateHydrator( + socialGraphServiceProcessStore: ReadableStore[RelationEdge, Boolean], + safeUserStore: ReadableStore[Long, User], + apiListStore: ReadableStore[Long, ApiList], + candidateCopyCross: CandidateCopyExpansion +)( + implicit statsReceiver: StatsReceiver, + implicit val weightedOpenOrNtabClickModelScorer: PushMLModelScorer) { + + lazy val candidateWithCopyNumStat = statsReceiver.stat("candidate_with_copy_num") + lazy val hydratedCandidateStat = statsReceiver.scope("hydrated_candidates") + lazy val mrUserStateStat = statsReceiver.scope("mr_user_state") + + lazy val queryStep = statsReceiver.scope("query_step") + lazy val relationEdgeWithoutDuplicateInQueryStep = + queryStep.counter("number_of_relationEdge_without_duplicate_in_query_step") + lazy val relationEdgeWithoutDuplicateInQueryStepDistribution = + queryStep.stat("number_of_relationEdge_without_duplicate_in_query_step_distribution") + + case class Entities( + users: Set[Long] = Set.empty[Long], + relationshipEdges: Set[RelationEdge] = Set.empty[RelationEdge]) { + def merge(otherEntities: Entities): Entities = { + this.copy( + users = this.users ++ otherEntities.users, + relationshipEdges = + this.relationshipEdges ++ otherEntities.relationshipEdges + ) + } + } + + case class EntitiesMap( + userMap: Map[Long, User] = Map.empty[Long, User], + relationshipMap: Map[RelationEdge, Boolean] = Map.empty[RelationEdge, Boolean]) + + private def updateCandidateAndCrtStats( + candidate: RawCandidate, + candidateType: String, + numEntities: Int = 1 + ): Unit = { + statsReceiver + .scope(candidateType).scope(candidate.commonRecType.name).stat( + "totalEntitiesPerCandidateTypePerCrt").add(numEntities) + statsReceiver.scope(candidateType).stat("totalEntitiesPerCandidateType").add(numEntities) + } + + private def collectEntities( + candidateDetailsSeq: Seq[CandidateDetails[RawCandidate]] + ): Entities = { + candidateDetailsSeq + .map { candidateDetails => + val pushCandidate = candidateDetails.candidate + + val userEntities = pushCandidate match { + case tweetWithSocialContext: RawCandidate with TweetWithSocialContextTraits => + val authorIdOpt = getAuthorIdFromTweetCandidate(tweetWithSocialContext) + val scUserIds = tweetWithSocialContext.socialContextUserIds.toSet + updateCandidateAndCrtStats(pushCandidate, "tweetWithSocialContext", scUserIds.size + 1) + Entities(users = scUserIds ++ authorIdOpt.toSet) + + case _ => Entities() + } + + val relationEntities = { + if (isInNetworkTweetType(pushCandidate.commonRecType)) { + Entities( + relationshipEdges = + RelationshipUtil.getPreCandidateRelationshipsForInNetworkTweets(pushCandidate).toSet + ) + } else Entities() + } + + userEntities.merge(relationEntities) + } + .foldLeft(Entities()) { (e1, e2) => e1.merge(e2) } + + } + + /** + * This method calls Gizmoduck and Social Graph Service, keep the results in EntitiesMap + * and passed onto the update candidate phase in the hydration step + * + * @param entities contains all userIds and relationEdges for all candidates + * @return EntitiesMap contains userMap and relationshipMap + */ + private def queryEntities(entities: Entities): Future[EntitiesMap] = { + + relationEdgeWithoutDuplicateInQueryStep.incr(entities.relationshipEdges.size) + relationEdgeWithoutDuplicateInQueryStepDistribution.add(entities.relationshipEdges.size) + + val relationshipMapFuture = Future + .collect(socialGraphServiceProcessStore.multiGet(entities.relationshipEdges)) + .map { resultMap => + resultMap.collect { + case (relationshipEdge, Some(res)) => relationshipEdge -> res + case (relationshipEdge, None) => relationshipEdge -> false + } + } + + val userMapFuture = Future + .collect(safeUserStore.multiGet(entities.users)) + .map { userMap => + userMap.collect { + case (userId, Some(user)) => + userId -> user + } + } + + Future.join(userMapFuture, relationshipMapFuture).map { + case (uMap, rMap) => EntitiesMap(userMap = uMap, relationshipMap = rMap) + } + } + + /** + * @param candidateDetails: recommendation candidates for a user + * @return sequence of candidates tagged with push and ntab copy id + */ + private def expandCandidatesWithCopy( + candidateDetails: Seq[CandidateDetails[RawCandidate]] + ): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = { + candidateCopyCross.expandCandidatesWithCopyId(candidateDetails) + } + + def updateCandidates( + candidateDetailsWithCopies: Seq[(CandidateDetails[RawCandidate], CopyIds)], + entitiesMaps: EntitiesMap + ): Seq[CandidateDetails[PushCandidate]] = { + candidateDetailsWithCopies.map { + case (candidateDetail, copyIds) => + val pushCandidate = candidateDetail.candidate + val userMap = entitiesMaps.userMap + val relationshipMap = entitiesMaps.relationshipMap + + val hydratedCandidate = pushCandidate match { + + case f1TweetCandidate: F1FirstDegree => + getHydratedCandidateForF1FirstDegreeTweet( + f1TweetCandidate, + userMap, + relationshipMap, + copyIds) + + case tweetRetweet: TweetRetweetCandidate => + getHydratedCandidateForTweetRetweet(tweetRetweet, userMap, copyIds) + + case tweetFavorite: TweetFavoriteCandidate => + getHydratedCandidateForTweetFavorite(tweetFavorite, userMap, copyIds) + + case tripTweetCandidate: OutOfNetworkTweetCandidate with TripCandidate => + getHydratedCandidateForTripTweetCandidate(tripTweetCandidate, userMap, copyIds) + + case outOfNetworkTweetCandidate: OutOfNetworkTweetCandidate with TopicCandidate => + getHydratedCandidateForOutOfNetworkTweetCandidate( + outOfNetworkTweetCandidate, + userMap, + copyIds) + + case topicProofTweetCandidate: TopicProofTweetCandidate => + getHydratedTopicProofTweetCandidate(topicProofTweetCandidate, userMap, copyIds) + + case subscribedSearchTweetCandidate: SubscribedSearchTweetCandidate => + getHydratedSubscribedSearchTweetCandidate( + subscribedSearchTweetCandidate, + userMap, + copyIds) + + case listRecommendation: ListPushCandidate => + getHydratedListCandidate(apiListStore, listRecommendation, copyIds) + + case discoverTwitterCandidate: DiscoverTwitterCandidate => + getHydratedCandidateForDiscoverTwitterCandidate(discoverTwitterCandidate, copyIds) + + case topTweetImpressionsCandidate: TopTweetImpressionsCandidate => + getHydratedCandidateForTopTweetImpressionsCandidate( + topTweetImpressionsCandidate, + copyIds) + + case trendTweetCandidate: TrendTweetCandidate => + new TrendTweetPushCandidate( + trendTweetCandidate, + trendTweetCandidate.authorId.flatMap(userMap.get), + copyIds) + + case unknownCandidate => + throw new IllegalArgumentException( + s"Incorrect candidate for hydration: ${unknownCandidate.commonRecType}") + } + + CandidateDetails( + hydratedCandidate, + source = candidateDetail.source + ) + } + } + + def apply( + candidateDetails: Seq[CandidateDetails[RawCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + val isLoggedOutRequest = + candidateDetails.headOption.exists(_.candidate.target.isLoggedOutUser) + if (!isLoggedOutRequest) { + candidateDetails.headOption.map { cd => + MrUserStateUtil.updateMrUserStateStats(cd.candidate.target)(mrUserStateStat) + } + } + + expandCandidatesWithCopy(candidateDetails).flatMap { candidateDetailsWithCopy => + candidateWithCopyNumStat.add(candidateDetailsWithCopy.size) + val entities = collectEntities(candidateDetailsWithCopy.map(_._1)) + queryEntities(entities).flatMap { entitiesMap => + val updatedCandidates = updateCandidates(candidateDetailsWithCopy, entitiesMap) + updatedCandidates.foreach { cand => + hydratedCandidateStat.counter(cand.candidate.commonRecType.name).incr() + } + Future.value(updatedCandidates) + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.scala new file mode 100644 index 0000000000..6d1172cb9c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.scala @@ -0,0 +1,69 @@ +package com.twitter.frigate.pushservice.refresh_handler + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.FeatureMap +import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.ml.HydrationContextBuilder +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.util.MrUserStateUtil +import com.twitter.nrel.heavyranker.FeatureHydrator +import com.twitter.util.Future + +class RFPHFeatureHydrator( + featureHydrator: FeatureHydrator +)( + implicit globalStats: StatsReceiver) { + + implicit val statsReceiver: StatsReceiver = + globalStats.scope("RefreshForPushHandler") + + //stat for feature hydration + private val featureHydrationEnabledCounter = statsReceiver.counter("featureHydrationEnabled") + private val mrUserStateStat = statsReceiver.scope("mr_user_state") + + private def hydrateFromRelevanceHydrator( + candidateDetails: Seq[CandidateDetails[PushCandidate]], + mrRequestContextForFeatureStore: MrRequestContextForFeatureStore + ): Future[Unit] = { + val pushCandidates = candidateDetails.map(_.candidate) + val candidatesAndContextsFut = Future.collect(pushCandidates.map { pc => + val contextFut = HydrationContextBuilder.build(pc) + contextFut.map { ctx => (pc, ctx) } + }) + candidatesAndContextsFut.flatMap { candidatesAndContexts => + val contexts = candidatesAndContexts.map(_._2) + val resultsFut = featureHydrator.hydrateCandidate(contexts, mrRequestContextForFeatureStore) + resultsFut.map { hydrationResult => + candidatesAndContexts.foreach { + case (pushCandidate, context) => + val resultFeatures = hydrationResult.getOrElse(context, FeatureMap()) + pushCandidate.mergeFeatures(resultFeatures) + } + } + } + } + + def candidateFeatureHydration( + candidateDetails: Seq[CandidateDetails[PushCandidate]], + mrRequestContextForFeatureStore: MrRequestContextForFeatureStore + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + candidateDetails.headOption match { + case Some(cand) => + val target = cand.candidate.target + MrUserStateUtil.updateMrUserStateStats(target)(mrUserStateStat) + if (target.params(PushParams.DisableAllRelevanceParam)) { + Future.value(candidateDetails) + } else { + featureHydrationEnabledCounter.incr() + for { + _ <- hydrateFromRelevanceHydrator(candidateDetails, mrRequestContextForFeatureStore) + } yield { + candidateDetails + } + } + case _ => Future.Nil + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.scala new file mode 100644 index 0000000000..fe52428b30 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.scala @@ -0,0 +1,104 @@ +package com.twitter.frigate.pushservice.refresh_handler + +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.predicate.PreRankingPredicates +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.SequentialPredicate +import com.twitter.util._ + +class RFPHPrerankFilter( +)( + globalStats: StatsReceiver) { + def filter( + target: Target, + hydratedCandidates: Seq[CandidateDetails[PushCandidate]] + ): Future[ + (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) + ] = { + lazy val filterStats: StatsReceiver = globalStats.scope("RefreshForPushHandler/filter") + lazy val okFilterCounter: Counter = filterStats.counter("ok") + lazy val invalidFilterCounter: Counter = filterStats.counter("invalid") + lazy val invalidFilterStat: StatsReceiver = filterStats.scope("invalid") + lazy val invalidFilterReasonStat: StatsReceiver = invalidFilterStat.scope("reason") + val allCandidatesFilteredPreRank = filterStats.counter("all_candidates_filtered") + + lazy val preRankingPredicates = PreRankingPredicates( + filterStats.scope("predicates") + ) + + lazy val preRankingPredicateChain = + new SequentialPredicate[PushCandidate](preRankingPredicates) + + val predicateChain = if (target.pushContext.exists(_.predicatesToEnable.exists(_.nonEmpty))) { + val predicatesToEnable = target.pushContext.flatMap(_.predicatesToEnable).getOrElse(Nil) + new SequentialPredicate[PushCandidate](preRankingPredicates.filter { pred => + predicatesToEnable.contains(pred.name) + }) + } else preRankingPredicateChain + + predicateChain + .track(hydratedCandidates.map(_.candidate)) + .map { results => + val resultForPreRankFiltering = results + .zip(hydratedCandidates) + .foldLeft( + ( + Seq.empty[CandidateDetails[PushCandidate]], + Seq.empty[CandidateResult[PushCandidate, Result]] + ) + ) { + case ((goodCandidates, filteredCandidates), (result, candidateDetails)) => + result match { + case None => + okFilterCounter.incr() + (goodCandidates :+ candidateDetails, filteredCandidates) + + case Some(pred: NamedPredicate[_]) => + invalidFilterCounter.incr() + invalidFilterReasonStat.counter(pred.name).incr() + invalidFilterReasonStat + .scope(candidateDetails.candidate.commonRecType.toString).counter( + pred.name).incr() + + val r = Invalid(Some(pred.name)) + ( + goodCandidates, + filteredCandidates :+ CandidateResult[PushCandidate, Result]( + candidateDetails.candidate, + candidateDetails.source, + r + ) + ) + case Some(_) => + invalidFilterCounter.incr() + invalidFilterReasonStat.counter("unknown").incr() + invalidFilterReasonStat + .scope(candidateDetails.candidate.commonRecType.toString).counter( + "unknown").incr() + + val r = Invalid(Some("Filtered by un-named predicate")) + ( + goodCandidates, + filteredCandidates :+ CandidateResult[PushCandidate, Result]( + candidateDetails.candidate, + candidateDetails.source, + r + ) + ) + } + } + + resultForPreRankFiltering match { + case (validCandidates, _) if validCandidates.isEmpty && hydratedCandidates.nonEmpty => + allCandidatesFilteredPreRank.incr() + case _ => () + } + + resultForPreRankFiltering + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.scala new file mode 100644 index 0000000000..0374791115 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.scala @@ -0,0 +1,34 @@ +package com.twitter.frigate.pushservice.refresh_handler + +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.TargetUser +import com.twitter.frigate.common.candidate.TargetABDecider +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.target.TargetScoringDetails + +class RFPHRestrictStep()(implicit stats: StatsReceiver) { + + private val statsReceiver: StatsReceiver = stats.scope("RefreshForPushHandler") + private val restrictStepStats: StatsReceiver = statsReceiver.scope("restrict") + private val restrictStepNumCandidatesDroppedStat: Stat = + restrictStepStats.stat("candidates_dropped") + + /** + * Limit the number of candidates that enter the Take step + */ + def restrict( + target: TargetUser with TargetABDecider with TargetScoringDetails, + candidates: Seq[CandidateDetails[PushCandidate]] + ): (Seq[CandidateDetails[PushCandidate]], Seq[CandidateDetails[PushCandidate]]) = { + if (target.params(PushFeatureSwitchParams.EnableRestrictStep)) { + val restrictSizeParam = PushFeatureSwitchParams.RestrictStepSize + val (newCandidates, filteredCandidates) = candidates.splitAt(target.params(restrictSizeParam)) + val numDropped = candidates.length - newCandidates.length + restrictStepNumCandidatesDroppedStat.add(numDropped) + (newCandidates, filteredCandidates) + } else (candidates, Seq.empty) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.scala new file mode 100644 index 0000000000..c09b4348a5 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.scala @@ -0,0 +1,77 @@ +package com.twitter.frigate.pushservice.refresh_handler + +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.thriftscala.CommonRecommendationType + +class RFPHStatsRecorder(implicit statsReceiver: StatsReceiver) { + + private val selectedCandidateScoreStats: StatsReceiver = + statsReceiver.scope("score_of_sent_candidate_times_10000") + + private val emptyScoreStats: StatsReceiver = + statsReceiver.scope("score_of_sent_candidate_empty") + + def trackPredictionScoreStats(candidate: PushCandidate): Unit = { + candidate.mrWeightedOpenOrNtabClickRankingProbability.foreach { + case Some(s) => + selectedCandidateScoreStats + .stat("weighted_open_or_ntab_click_ranking") + .add((s * 10000).toFloat) + case None => + emptyScoreStats.counter("weighted_open_or_ntab_click_ranking").incr() + } + candidate.mrWeightedOpenOrNtabClickFilteringProbability.foreach { + case Some(s) => + selectedCandidateScoreStats + .stat("weighted_open_or_ntab_click_filtering") + .add((s * 10000).toFloat) + case None => + emptyScoreStats.counter("weighted_open_or_ntab_click_filtering").incr() + } + candidate.mrWeightedOpenOrNtabClickRankingProbability.foreach { + case Some(s) => + selectedCandidateScoreStats + .scope(candidate.commonRecType.toString) + .stat("weighted_open_or_ntab_click_ranking") + .add((s * 10000).toFloat) + case None => + emptyScoreStats + .scope(candidate.commonRecType.toString) + .counter("weighted_open_or_ntab_click_ranking") + .incr() + } + } + + def refreshRequestExceptionStats( + exception: Throwable, + bStats: StatsReceiver + ): Unit = { + bStats.counter("failures").incr() + bStats.scope("failures").counter(exception.getClass.getCanonicalName).incr() + } + + def loggedOutRequestExceptionStats( + exception: Throwable, + bStats: StatsReceiver + ): Unit = { + bStats.counter("logged_out_failures").incr() + bStats.scope("failures").counter(exception.getClass.getCanonicalName).incr() + } + + def rankDistributionStats( + candidatesDetails: Seq[CandidateDetails[PushCandidate]], + numRecsPerTypeStat: (CommonRecommendationType => Stat) + ): Unit = { + candidatesDetails + .groupBy { c => + c.candidate.commonRecType + } + .mapValues { s => + s.size + } + .foreach { case (crt, numRecs) => numRecsPerTypeStat(crt).add(numRecs) } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.scala new file mode 100644 index 0000000000..17fb846cfe --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.scala @@ -0,0 +1,292 @@ +package com.twitter.frigate.pushservice.refresh_handler + +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.Stats.track +import com.twitter.frigate.common.base.Stats.trackSeq +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.logger.MRLogger +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.adaptor._ +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.rank.RFPHLightRanker +import com.twitter.frigate.pushservice.rank.RFPHRanker +import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler +import com.twitter.frigate.pushservice.take.candidate_validator.RFPHCandidateValidator +import com.twitter.frigate.pushservice.target.PushTargetUserBuilder +import com.twitter.frigate.pushservice.target.RFPHTargetPredicates +import com.twitter.frigate.pushservice.util.RFPHTakeStepUtil +import com.twitter.frigate.pushservice.util.AdhocStatsUtil +import com.twitter.frigate.pushservice.thriftscala.PushContext +import com.twitter.frigate.pushservice.thriftscala.RefreshRequest +import com.twitter.frigate.pushservice.thriftscala.RefreshResponse +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.hermit.predicate.Predicate +import com.twitter.timelines.configapi.FeatureValue +import com.twitter.util._ + +case class ResultWithDebugInfo(result: Result, predicateResults: Seq[PredicateWithResult]) + +class RefreshForPushHandler( + val pushTargetUserBuilder: PushTargetUserBuilder, + val candSourceGenerator: PushCandidateSourceGenerator, + rfphRanker: RFPHRanker, + candidateHydrator: PushCandidateHydrator, + candidateValidator: RFPHCandidateValidator, + rfphTakeStepUtil: RFPHTakeStepUtil, + rfphRestrictStep: RFPHRestrictStep, + val rfphNotifier: RefreshForPushNotifier, + rfphStatsRecorder: RFPHStatsRecorder, + mrRequestScriberNode: String, + rfphFeatureHydrator: RFPHFeatureHydrator, + rfphPrerankFilter: RFPHPrerankFilter, + rfphLightRanker: RFPHLightRanker +)( + globalStats: StatsReceiver) + extends FetchRankFlowWithHydratedCandidates[Target, RawCandidate, PushCandidate] { + + val log = MRLogger("RefreshForPushHandler") + + implicit val statsReceiver: StatsReceiver = + globalStats.scope("RefreshForPushHandler") + private val maxCandidatesToBatchInTakeStat: Stat = + statsReceiver.stat("max_cands_to_batch_in_take") + + private val rfphRequestCounter = statsReceiver.counter("requests") + + private val buildTargetStats = statsReceiver.scope("build_target") + private val processStats = statsReceiver.scope("process") + private val notifyStats = statsReceiver.scope("notify") + + private val lightRankingStats: StatsReceiver = statsReceiver.scope("light_ranking") + private val reRankingStats: StatsReceiver = statsReceiver.scope("rerank") + private val featureHydrationLatency: StatsReceiver = + statsReceiver.scope("featureHydrationLatency") + private val candidateHydrationStats: StatsReceiver = statsReceiver.scope("candidate_hydration") + + lazy val candSourceEligibleCounter: Counter = + candidateStats.counter("cand_source_eligible") + lazy val candSourceNotEligibleCounter: Counter = + candidateStats.counter("cand_source_not_eligible") + + //pre-ranking stats + val allCandidatesFilteredPreRank = filterStats.counter("all_candidates_filtered") + + // total invalid candidates + val totalStats: StatsReceiver = statsReceiver.scope("total") + val totalInvalidCandidatesStat: Stat = totalStats.stat("candidates_invalid") + + val mrRequestScribeBuiltStats: Counter = statsReceiver.counter("mr_request_scribe_built") + + val mrRequestCandidateScribeStats = statsReceiver.scope("mr_request_scribe_candidates") + val mrRequestTargetScribeStats = statsReceiver.scope("mr_request_scribe_target") + + val mrRequestScribeHandler = + new MrRequestScribeHandler(mrRequestScriberNode, statsReceiver.scope("mr_request_scribe")) + + val adhocStatsUtil = new AdhocStatsUtil(statsReceiver.scope("adhoc_stats")) + + private def numRecsPerTypeStat(crt: CommonRecommendationType) = + fetchStats.scope(crt.toString).stat("dist") + + // static list of target predicates + private val targetPredicates = RFPHTargetPredicates(targetStats.scope("predicates")) + + def buildTarget( + userId: Long, + inputPushContext: Option[PushContext], + forcedFeatureValues: Option[Map[String, FeatureValue]] = None + ): Future[Target] = + pushTargetUserBuilder.buildTarget(userId, inputPushContext, forcedFeatureValues) + + override def targetPredicates(target: Target): List[Predicate[Target]] = targetPredicates + + override def isTargetValid(target: Target): Future[Result] = { + val resultFut = if (target.skipFilters) { + Future.value(trackTargetPredStats(None)) + } else { + predicateSeq(target).track(Seq(target)).map { resultArr => + trackTargetPredStats(resultArr(0)) + } + } + track(targetStats)(resultFut) + } + + override def candidateSources( + target: Target + ): Future[Seq[CandidateSource[Target, RawCandidate]]] = { + Future + .collect(candSourceGenerator.sources.map { cs => + cs.isCandidateSourceAvailable(target).map { isEligible => + if (isEligible) { + candSourceEligibleCounter.incr() + Some(cs) + } else { + candSourceNotEligibleCounter.incr() + None + } + } + }).map(_.flatten) + } + + override def updateCandidateCounter( + candidateResults: Seq[CandidateResult[PushCandidate, Result]] + ): Unit = { + candidateResults.foreach { + case candidateResult if candidateResult.result == OK => + okCandidateCounter.incr() + case candidateResult if candidateResult.result.isInstanceOf[Invalid] => + invalidCandidateCounter.incr() + case _ => + } + } + + override def hydrateCandidates( + candidates: Seq[CandidateDetails[RawCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = candidateHydrator(candidates) + + override def filter( + target: Target, + hydratedCandidates: Seq[CandidateDetails[PushCandidate]] + ): Future[ + (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) + ] = rfphPrerankFilter.filter(target, hydratedCandidates) + + def lightRankAndTake( + target: Target, + candidates: Seq[CandidateDetails[PushCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + rfphLightRanker.rank(target, candidates) + } + + override def rank( + target: Target, + candidatesDetails: Seq[CandidateDetails[PushCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + val featureHydratedCandidatesFut = trackSeq(featureHydrationLatency)( + rfphFeatureHydrator + .candidateFeatureHydration(candidatesDetails, target.mrRequestContextForFeatureStore) + ) + featureHydratedCandidatesFut.flatMap { featureHydratedCandidates => + rfphStatsRecorder.rankDistributionStats(featureHydratedCandidates, numRecsPerTypeStat) + rfphRanker.initialRank(target, candidatesDetails) + } + } + + def reRank( + target: Target, + rankedCandidates: Seq[CandidateDetails[PushCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + rfphRanker.reRank(target, rankedCandidates) + } + + override def validCandidates( + target: Target, + candidates: Seq[PushCandidate] + ): Future[Seq[Result]] = { + Future.collect(candidates.map { candidate => + rfphTakeStepUtil.isCandidateValid(candidate, candidateValidator).map(res => res.result) + }) + } + + override def desiredCandidateCount(target: Target): Int = target.desiredCandidateCount + + override def batchForCandidatesCheck(target: Target): Int = { + val fsParam = PushFeatureSwitchParams.NumberOfMaxCandidatesToBatchInRFPHTakeStep + val maxToBatch = target.params(fsParam) + maxCandidatesToBatchInTakeStat.add(maxToBatch) + maxToBatch + } + + override def process( + target: Target, + externalCandidates: Seq[RawCandidate] = Nil + ): Future[Response[PushCandidate, Result]] = { + isTargetValid(target).flatMap { + case OK => + for { + candidatesFromSources <- trackSeq(fetchStats)(fetchCandidates(target)) + externalCandidateDetails = externalCandidates.map( + CandidateDetails(_, "refresh_for_push_handler_external_candidate")) + allCandidates = candidatesFromSources ++ externalCandidateDetails + hydratedCandidatesWithCopy <- + trackSeq(candidateHydrationStats)(hydrateCandidates(allCandidates)) + _ = adhocStatsUtil.getCandidateSourceStats(hydratedCandidatesWithCopy) + (candidates, preRankingFilteredCandidates) <- + track(filterStats)(filter(target, hydratedCandidatesWithCopy)) + _ = adhocStatsUtil.getPreRankingFilterStats(preRankingFilteredCandidates) + lightRankerFilteredCandidates <- + trackSeq(lightRankingStats)(lightRankAndTake(target, candidates)) + _ = adhocStatsUtil.getLightRankingStats(lightRankerFilteredCandidates) + rankedCandidates <- trackSeq(rankingStats)(rank(target, lightRankerFilteredCandidates)) + _ = adhocStatsUtil.getRankingStats(rankedCandidates) + rerankedCandidates <- trackSeq(reRankingStats)(reRank(target, rankedCandidates)) + _ = adhocStatsUtil.getReRankingStats(rerankedCandidates) + (restrictedCandidates, restrictFilteredCandidates) = + rfphRestrictStep.restrict(target, rerankedCandidates) + allTakeCandidateResults <- track(takeStats)( + take(target, restrictedCandidates, desiredCandidateCount(target)) + ) + _ = adhocStatsUtil.getTakeCandidateResultStats(allTakeCandidateResults) + _ <- track(mrRequestCandidateScribeStats)( + mrRequestScribeHandler.scribeForCandidateFiltering( + target, + hydratedCandidatesWithCopy, + preRankingFilteredCandidates, + rankedCandidates, + rerankedCandidates, + restrictFilteredCandidates, + allTakeCandidateResults + )) + } yield { + + /** + * Take processes post restrict step candidates and returns both: + * 1. valid + invalid candidates + * 2. Candidates that are not processed (more than desired) + restricted candidates + * We need #2 only for importance sampling + */ + val takeCandidateResults = + allTakeCandidateResults.filterNot { candResult => + candResult.result == MoreThanDesiredCandidates + } + + val totalInvalidCandidates = { + preRankingFilteredCandidates.size + //pre-ranking filtered candidates + (rerankedCandidates.length - restrictedCandidates.length) + //candidates reject in restrict step + takeCandidateResults.count(_.result != OK) //candidates reject in take step + } + takeInvalidCandidateDist.add( + takeCandidateResults + .count(_.result != OK) + ) // take step invalid candidates + totalInvalidCandidatesStat.add(totalInvalidCandidates) + val allCandidateResults = takeCandidateResults ++ preRankingFilteredCandidates + Response(OK, allCandidateResults) + } + + case result: Result => + for (_ <- track(mrRequestTargetScribeStats)( + mrRequestScribeHandler.scribeForTargetFiltering(target, result))) yield { + mrRequestScribeBuiltStats.incr() + Response(result, Nil) + } + } + } + + def refreshAndSend(request: RefreshRequest): Future[RefreshResponse] = { + rfphRequestCounter.incr() + for { + target <- track(buildTargetStats)( + pushTargetUserBuilder + .buildTarget(request.userId, request.context)) + response <- track(processStats)(process(target, externalCandidates = Seq.empty)) + refreshResponse <- track(notifyStats)(rfphNotifier.checkResponseAndNotify(response, target)) + } yield { + refreshResponse + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.scala new file mode 100644 index 0000000000..ae68d46eac --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.scala @@ -0,0 +1,128 @@ +package com.twitter.frigate.pushservice.refresh_handler + +import com.twitter.finagle.stats.BroadcastStatsReceiver +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.Stats.track +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.config.CommonConstants +import com.twitter.frigate.common.util.PushServiceUtil.FilteredRefreshResponseFut +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.take.CandidateNotifier +import com.twitter.frigate.pushservice.util.ResponseStatsTrackUtils.trackStatsForResponseToRequest +import com.twitter.frigate.pushservice.thriftscala.PushStatus +import com.twitter.frigate.pushservice.thriftscala.RefreshResponse +import com.twitter.util.Future +import com.twitter.util.JavaTimer +import com.twitter.util.Timer + +class RefreshForPushNotifier( + rfphStatsRecorder: RFPHStatsRecorder, + candidateNotifier: CandidateNotifier +)( + globalStats: StatsReceiver) { + + private implicit val statsReceiver: StatsReceiver = + globalStats.scope("RefreshForPushHandler") + + private val pushStats: StatsReceiver = statsReceiver.scope("push") + private val sendLatency: StatsReceiver = statsReceiver.scope("send_handler") + implicit private val timer: Timer = new JavaTimer(true) + + private def notify( + candidatesResult: CandidateResult[PushCandidate, Result], + target: Target, + receivers: Seq[StatsReceiver] + ): Future[RefreshResponse] = { + + val candidate = candidatesResult.candidate + + val predsResult = candidatesResult.result + + if (predsResult != OK) { + val invalidResult = predsResult + invalidResult match { + case Invalid(Some(reason)) => + Future.value(RefreshResponse(PushStatus.Filtered, Some(reason))) + case _ => + Future.value(RefreshResponse(PushStatus.Filtered, None)) + } + } else { + rfphStatsRecorder.trackPredictionScoreStats(candidate) + + val isQualityUprankingCandidate = candidate.mrQualityUprankingBoost.isDefined + val commonRecTypeStats = Seq( + statsReceiver.scope(candidate.commonRecType.toString), + globalStats.scope(candidate.commonRecType.toString) + ) + val qualityUprankingStats = Seq( + statsReceiver.scope("QualityUprankingCandidates").scope(candidate.commonRecType.toString), + globalStats.scope("QualityUprankingCandidates").scope(candidate.commonRecType.toString) + ) + + val receiversWithRecTypeStats = { + if (isQualityUprankingCandidate) { + receivers ++ commonRecTypeStats ++ qualityUprankingStats + } else { + receivers ++ commonRecTypeStats + } + } + track(sendLatency)(candidateNotifier.notify(candidate).map { res => + trackStatsForResponseToRequest( + candidate.commonRecType, + candidate.target, + res, + receiversWithRecTypeStats + )(globalStats) + RefreshResponse(res.status) + }) + } + } + + def checkResponseAndNotify( + response: Response[PushCandidate, Result], + targetUserContext: Target + ): Future[RefreshResponse] = { + val receivers = Seq(statsReceiver) + val refreshResponse = response match { + case Response(OK, processedCandidates) => + // valid rec candidates + val validCandidates = processedCandidates.filter(_.result == OK) + + // top rec candidate + validCandidates.headOption match { + case Some(candidatesResult) => + candidatesResult.result match { + case OK => + notify(candidatesResult, targetUserContext, receivers) + .onSuccess { nr => + pushStats.scope("result").counter(nr.status.name).incr() + } + case _ => + targetUserContext.isTeamMember.flatMap { isTeamMember => + FilteredRefreshResponseFut + } + } + case _ => + FilteredRefreshResponseFut + } + case Response(Invalid(reason), _) => + // invalid target with known reason + FilteredRefreshResponseFut.map(_.copy(targetFilteredBy = reason)) + case _ => + // invalid target + FilteredRefreshResponseFut + } + + val bStats = BroadcastStatsReceiver(receivers) + Stat + .timeFuture(bStats.stat("latency"))( + refreshResponse + .raiseWithin(CommonConstants.maxPushRequestDuration) + ) + .onFailure { exception => + rfphStatsRecorder.refreshRequestExceptionStats(exception, bStats) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.scala new file mode 100644 index 0000000000..47426a3864 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.scala @@ -0,0 +1,79 @@ +package com.twitter.frigate.pushservice.refresh_handler.cross + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.util.MRNtabCopy +import com.twitter.frigate.common.util.MRPushCopy +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.util.Future + +abstract class BaseCopyFramework(statsReceiver: StatsReceiver) { + + private val NoAvailableCopyStat = statsReceiver.scope("no_copy_for_crt") + private val NoAvailableNtabCopyStat = statsReceiver.scope("no_ntab_copy") + + /** + * Instantiate push copy filters + */ + protected final val copyFilters = new CopyFilters(statsReceiver.scope("filters")) + + /** + * + * The following method fetches all the push copies for a [[com.twitter.frigate.thriftscala.CommonRecommendationType]] + * associated with a candidate and then filters the eligible copies based on + * [[PushTypes.PushCandidate]] features. These filters are defined in + * [[CopyFilters]] + * + * @param rawCandidate - [[RawCandidate]] object representing a recommendation candidate + * + * @return - set of eligible push copies for a given candidate + */ + protected[cross] final def getEligiblePushCopiesFromCandidate( + rawCandidate: RawCandidate + ): Future[Seq[MRPushCopy]] = { + val pushCopiesFromRectype = CandidateToCopy.getPushCopiesFromRectype(rawCandidate.commonRecType) + + if (pushCopiesFromRectype.isEmpty) { + NoAvailableCopyStat.counter(rawCandidate.commonRecType.name).incr() + throw new IllegalStateException(s"No Copy defined for CRT: " + rawCandidate.commonRecType) + } + pushCopiesFromRectype + .map(pushCopySet => copyFilters.execute(rawCandidate, pushCopySet.toSeq)) + .getOrElse(Future.value(Seq.empty)) + } + + /** + * + * This method essentially forms the base for cross-step for the MagicRecs Copy Framework. Given + * a recommendation type this returns a set of tuples wherein each tuple is a pair of push and + * ntab copy eligible for the said recommendation type + * + * @param rawCandidate - [[RawCandidate]] object representing a recommendation candidate + * @return - Set of eligible [[MRPushCopy]], Option[[MRNtabCopy]] for a given recommendation type + */ + protected[cross] final def getEligiblePushAndNtabCopiesFromCandidate( + rawCandidate: RawCandidate + ): Future[Seq[(MRPushCopy, Option[MRNtabCopy])]] = { + + val eligiblePushCopies = getEligiblePushCopiesFromCandidate(rawCandidate) + + eligiblePushCopies.map { pushCopies => + val setBuilder = Set.newBuilder[(MRPushCopy, Option[MRNtabCopy])] + pushCopies.foreach { pushCopy => + val ntabCopies = CandidateToCopy.getNtabcopiesFromPushcopy(pushCopy) + val pushNtabCopyPairs = ntabCopies match { + case Some(ntabCopySet) => + if (ntabCopySet.isEmpty) { + NoAvailableNtabCopyStat.counter(s"copy_id: ${pushCopy.copyId}").incr() + Set(pushCopy -> None) + } // push copy only + else ntabCopySet.map(pushCopy -> Some(_)) + + case None => + Set.empty[(MRPushCopy, Option[MRNtabCopy])] // no push or ntab copy + } + setBuilder ++= pushNtabCopyPairs + } + setBuilder.result().toSeq + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.scala new file mode 100644 index 0000000000..9748c90ff7 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.scala @@ -0,0 +1,56 @@ +package com.twitter.frigate.pushservice.refresh_handler.cross + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.util.MRNtabCopy +import com.twitter.frigate.common.util.MRPushCopy +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.util.Future + +/** + * @param statsReceiver - stats receiver object + */ +class CandidateCopyExpansion(statsReceiver: StatsReceiver) + extends BaseCopyFramework(statsReceiver) { + + /** + * + * Given a [[CandidateDetails]] object representing a push recommendation candidate this method + * expands it to multiple candidates, each tagged with a push copy id and ntab copy id to + * represent the eligible copies for the given recommendation candidate + * + * @param candidateDetails - [[CandidateDetails]] objects containing a recommendation candidate + * + * @return - list of tuples of [[PushTypes.RawCandidate]] and [[CopyIds]] + */ + private final def crossCandidateDetailsWithCopyId( + candidateDetails: CandidateDetails[RawCandidate] + ): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = { + val eligibleCopyPairs = getEligiblePushAndNtabCopiesFromCandidate(candidateDetails.candidate) + val copyPairs = eligibleCopyPairs.map(_.map { + case (pushCopy: MRPushCopy, ntabCopy: Option[MRNtabCopy]) => + CopyIds( + pushCopyId = Some(pushCopy.copyId), + ntabCopyId = ntabCopy.map(_.copyId) + ) + }) + + copyPairs.map(_.map((candidateDetails, _))) + } + + /** + * + * This method takes as input a list of [[CandidateDetails]] objects which contain the push + * recommendation candidates for a given target user. It expands each input candidate into + * multiple candidates, each tagged with a push copy id and ntab copy id to represent the eligible + * copies for the given recommendation candidate + * + * @param candidateDetailsSeq - list of fetched candidates for push recommendation + * @return - list of tuples of [[RawCandidate]] and [[CopyIds]] + */ + final def expandCandidatesWithCopyId( + candidateDetailsSeq: Seq[CandidateDetails[RawCandidate]] + ): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = + Future.collect(candidateDetailsSeq.map(crossCandidateDetailsWithCopyId)).map(_.flatten) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.scala new file mode 100644 index 0000000000..4eca417302 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.scala @@ -0,0 +1,11 @@ +package com.twitter.frigate.pushservice.refresh_handler.cross + +import com.twitter.frigate.common.util.MRPushCopy +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate + +/** + * + * @param candidate: [[RawCandidate]] is a recommendation candidate + * @param pushCopy: [[MRPushCopy]] eligible for candidate + */ +case class CandidateCopyPair(candidate: RawCandidate, pushCopy: MRPushCopy) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.scala new file mode 100644 index 0000000000..e7fbefe164 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.scala @@ -0,0 +1,263 @@ +package com.twitter.frigate.pushservice.refresh_handler.cross + +import com.twitter.frigate.common.util.MrNtabCopyObjects +import com.twitter.frigate.common.util.MrPushCopyObjects +import com.twitter.frigate.common.util._ +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.CommonRecommendationType._ + +object CandidateToCopy { + + // Static map from a CommonRecommendationType to set of eligible push notification copies + private[cross] val rectypeToPushCopy: Map[CommonRecommendationType, Set[ + MRPushCopy + ]] = + Map[CommonRecommendationType, Set[MRPushCopy]]( + F1FirstdegreeTweet -> Set( + MrPushCopyObjects.FirstDegreeJustTweetedBoldTitle + ), + F1FirstdegreePhoto -> Set( + MrPushCopyObjects.FirstDegreePhotoJustTweetedBoldTitle + ), + F1FirstdegreeVideo -> Set( + MrPushCopyObjects.FirstDegreeVideoJustTweetedBoldTitle + ), + TweetRetweet -> Set( + MrPushCopyObjects.TweetRetweetWithOneDisplaySocialContextsWithText, + MrPushCopyObjects.TweetRetweetWithTwoDisplaySocialContextsWithText, + MrPushCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContextsWithText + ), + TweetRetweetPhoto -> Set( + MrPushCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContextWithText, + MrPushCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContextsWithText, + MrPushCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContextsWithText + ), + TweetRetweetVideo -> Set( + MrPushCopyObjects.TweetRetweetVideoWithOneDisplaySocialContextWithText, + MrPushCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContextsWithText, + MrPushCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContextsWithText + ), + TweetFavorite -> Set( + MrPushCopyObjects.TweetLikeOneSocialContextWithText, + MrPushCopyObjects.TweetLikeTwoSocialContextWithText, + MrPushCopyObjects.TweetLikeMultipleSocialContextWithText + ), + TweetFavoritePhoto -> Set( + MrPushCopyObjects.TweetLikePhotoOneSocialContextWithText, + MrPushCopyObjects.TweetLikePhotoTwoSocialContextWithText, + MrPushCopyObjects.TweetLikePhotoMultipleSocialContextWithText + ), + TweetFavoriteVideo -> Set( + MrPushCopyObjects.TweetLikeVideoOneSocialContextWithText, + MrPushCopyObjects.TweetLikeVideoTwoSocialContextWithText, + MrPushCopyObjects.TweetLikeVideoMultipleSocialContextWithText + ), + UnreadBadgeCount -> Set(MrPushCopyObjects.UnreadBadgeCount), + InterestBasedTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + InterestBasedPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), + InterestBasedVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), + UserFollow -> Set( + MrPushCopyObjects.UserFollowWithOneSocialContext, + MrPushCopyObjects.UserFollowWithTwoSocialContext, + MrPushCopyObjects.UserFollowOneDisplayAndKOtherSocialContext + ), + HermitUser -> Set( + MrPushCopyObjects.HermitUserWithOneSocialContext, + MrPushCopyObjects.HermitUserWithTwoSocialContext, + MrPushCopyObjects.HermitUserWithOneDisplayAndKOtherSocialContexts + ), + TriangularLoopUser -> Set( + MrPushCopyObjects.TriangularLoopUserWithOneSocialContext, + MrPushCopyObjects.TriangularLoopUserWithTwoSocialContexts, + MrPushCopyObjects.TriangularLoopUserOneDisplayAndKotherSocialContext + ), + ForwardAddressbookUserFollow -> Set(MrPushCopyObjects.ForwardAddressBookUserFollow), + NewsArticleNewsLanding -> Set(MrPushCopyObjects.NewsArticleNewsLandingCopy), + TopicProofTweet -> Set(MrPushCopyObjects.TopicProofTweet), + UserInterestinTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + UserInterestinPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), + UserInterestinVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), + TwistlyTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + TwistlyPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), + TwistlyVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), + ElasticTimelineTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + ElasticTimelinePhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), + ElasticTimelineVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), + ExploreVideoTweet -> Set(MrPushCopyObjects.ExploreVideoTweet), + List -> Set(MrPushCopyObjects.ListRecommendation), + InterestBasedUserFollow -> Set(MrPushCopyObjects.UserFollowInterestBasedCopy), + PastEmailEngagementTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + PastEmailEngagementPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), + PastEmailEngagementVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), + ExplorePush -> Set(MrPushCopyObjects.ExplorePush), + ConnectTabPush -> Set(MrPushCopyObjects.ConnectTabPush), + ConnectTabWithUserPush -> Set(MrPushCopyObjects.ConnectTabWithUserPush), + AddressBookUploadPush -> Set(MrPushCopyObjects.AddressBookPush), + InterestPickerPush -> Set(MrPushCopyObjects.InterestPickerPush), + CompleteOnboardingPush -> Set(MrPushCopyObjects.CompleteOnboardingPush), + GeoPopTweet -> Set(MrPushCopyObjects.GeoPopPushCopy), + TagSpaceTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + FrsTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + TwhinTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + MrModelingBasedTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + DetopicTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + TweetImpressions -> Set(MrPushCopyObjects.TopTweetImpressions), + TrendTweet -> Set(MrPushCopyObjects.TrendTweet), + ReverseAddressbookTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + ForwardAddressbookTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + SpaceInNetwork -> Set(MrPushCopyObjects.SpaceHost), + SpaceOutOfNetwork -> Set(MrPushCopyObjects.SpaceHost), + SubscribedSearch -> Set(MrPushCopyObjects.SubscribedSearchTweet), + TripGeoTweet -> Set(MrPushCopyObjects.TripGeoTweetPushCopy), + CrowdSearchTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), + Digest -> Set(MrPushCopyObjects.Digest), + TripHqTweet -> Set(MrPushCopyObjects.TripHqTweetPushCopy) + ) + + // Static map from a push copy to set of eligible ntab copies + private[cross] val pushcopyToNtabcopy: Map[MRPushCopy, Set[MRNtabCopy]] = + Map[MRPushCopy, Set[MRNtabCopy]]( + MrPushCopyObjects.FirstDegreeJustTweetedBoldTitle -> Set( + MrNtabCopyObjects.FirstDegreeTweetRecent), + MrPushCopyObjects.FirstDegreePhotoJustTweetedBoldTitle -> Set( + MrNtabCopyObjects.FirstDegreeTweetRecent + ), + MrPushCopyObjects.FirstDegreeVideoJustTweetedBoldTitle -> Set( + MrNtabCopyObjects.FirstDegreeTweetRecent + ), + MrPushCopyObjects.TweetRetweetWithOneDisplaySocialContextsWithText -> Set( + MrNtabCopyObjects.TweetRetweetWithOneDisplaySocialContext + ), + MrPushCopyObjects.TweetRetweetWithTwoDisplaySocialContextsWithText -> Set( + MrNtabCopyObjects.TweetRetweetWithTwoDisplaySocialContexts + ), + MrPushCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContextsWithText -> Set( + MrNtabCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContexts + ), + MrPushCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContextWithText -> Set( + MrNtabCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContext + ), + MrPushCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContextsWithText -> Set( + MrNtabCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContexts + ), + MrPushCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContextsWithText -> Set( + MrNtabCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContexts + ), + MrPushCopyObjects.TweetRetweetVideoWithOneDisplaySocialContextWithText -> Set( + MrNtabCopyObjects.TweetRetweetVideoWithOneDisplaySocialContext + ), + MrPushCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContextsWithText -> Set( + MrNtabCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContexts + ), + MrPushCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContextsWithText -> Set( + MrNtabCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContexts + ), + MrPushCopyObjects.TweetLikeOneSocialContextWithText -> Set( + MrNtabCopyObjects.TweetLikeWithOneDisplaySocialContext + ), + MrPushCopyObjects.TweetLikeTwoSocialContextWithText -> Set( + MrNtabCopyObjects.TweetLikeWithTwoDisplaySocialContexts + ), + MrPushCopyObjects.TweetLikeMultipleSocialContextWithText -> Set( + MrNtabCopyObjects.TweetLikeWithOneDisplayAndKOtherSocialContexts + ), + MrPushCopyObjects.TweetLikePhotoOneSocialContextWithText -> Set( + MrNtabCopyObjects.TweetLikePhotoWithOneDisplaySocialContext + ), + MrPushCopyObjects.TweetLikePhotoTwoSocialContextWithText -> Set( + MrNtabCopyObjects.TweetLikePhotoWithTwoDisplaySocialContexts + ), + MrPushCopyObjects.TweetLikePhotoMultipleSocialContextWithText -> Set( + MrNtabCopyObjects.TweetLikePhotoWithOneDisplayAndKOtherSocialContexts + ), + MrPushCopyObjects.TweetLikeVideoOneSocialContextWithText -> Set( + MrNtabCopyObjects.TweetLikeVideoWithOneDisplaySocialContext + ), + MrPushCopyObjects.TweetLikeVideoTwoSocialContextWithText -> Set( + MrNtabCopyObjects.TweetLikeVideoWithTwoDisplaySocialContexts + ), + MrPushCopyObjects.TweetLikeVideoMultipleSocialContextWithText -> Set( + MrNtabCopyObjects.TweetLikeVideoWithOneDisplayAndKOtherSocialContexts + ), + MrPushCopyObjects.UnreadBadgeCount -> Set.empty[MRNtabCopy], + MrPushCopyObjects.RecommendedForYouTweet -> Set(MrNtabCopyObjects.RecommendedForYouCopy), + MrPushCopyObjects.RecommendedForYouPhoto -> Set(MrNtabCopyObjects.RecommendedForYouCopy), + MrPushCopyObjects.RecommendedForYouVideo -> Set(MrNtabCopyObjects.RecommendedForYouCopy), + MrPushCopyObjects.GeoPopPushCopy -> Set(MrNtabCopyObjects.RecommendedForYouCopy), + MrPushCopyObjects.UserFollowWithOneSocialContext -> Set( + MrNtabCopyObjects.UserFollowWithOneDisplaySocialContext + ), + MrPushCopyObjects.UserFollowWithTwoSocialContext -> Set( + MrNtabCopyObjects.UserFollowWithTwoDisplaySocialContexts + ), + MrPushCopyObjects.UserFollowOneDisplayAndKOtherSocialContext -> Set( + MrNtabCopyObjects.UserFollowWithOneDisplayAndKOtherSocialContexts + ), + MrPushCopyObjects.HermitUserWithOneSocialContext -> Set( + MrNtabCopyObjects.UserFollowWithOneDisplaySocialContext + ), + MrPushCopyObjects.HermitUserWithTwoSocialContext -> Set( + MrNtabCopyObjects.UserFollowWithTwoDisplaySocialContexts + ), + MrPushCopyObjects.HermitUserWithOneDisplayAndKOtherSocialContexts -> Set( + MrNtabCopyObjects.UserFollowWithOneDisplayAndKOtherSocialContexts + ), + MrPushCopyObjects.TriangularLoopUserWithOneSocialContext -> Set( + MrNtabCopyObjects.TriangularLoopUserWithOneSocialContext + ), + MrPushCopyObjects.TriangularLoopUserWithTwoSocialContexts -> Set( + MrNtabCopyObjects.TriangularLoopUserWithTwoSocialContexts + ), + MrPushCopyObjects.TriangularLoopUserOneDisplayAndKotherSocialContext -> Set( + MrNtabCopyObjects.TriangularLoopUserOneDisplayAndKOtherSocialContext + ), + MrPushCopyObjects.NewsArticleNewsLandingCopy -> Set( + MrNtabCopyObjects.NewsArticleNewsLandingCopy + ), + MrPushCopyObjects.UserFollowInterestBasedCopy -> Set( + MrNtabCopyObjects.UserFollowInterestBasedCopy + ), + MrPushCopyObjects.ForwardAddressBookUserFollow -> Set( + MrNtabCopyObjects.ForwardAddressBookUserFollow), + MrPushCopyObjects.ConnectTabPush -> Set( + MrNtabCopyObjects.ConnectTabPush + ), + MrPushCopyObjects.ExplorePush -> Set.empty[MRNtabCopy], + MrPushCopyObjects.ConnectTabWithUserPush -> Set( + MrNtabCopyObjects.UserFollowInterestBasedCopy), + MrPushCopyObjects.AddressBookPush -> Set(MrNtabCopyObjects.AddressBook), + MrPushCopyObjects.InterestPickerPush -> Set(MrNtabCopyObjects.InterestPicker), + MrPushCopyObjects.CompleteOnboardingPush -> Set(MrNtabCopyObjects.CompleteOnboarding), + MrPushCopyObjects.TopicProofTweet -> Set(MrNtabCopyObjects.TopicProofTweet), + MrPushCopyObjects.TopTweetImpressions -> Set(MrNtabCopyObjects.TopTweetImpressions), + MrPushCopyObjects.TrendTweet -> Set(MrNtabCopyObjects.TrendTweet), + MrPushCopyObjects.SpaceHost -> Set(MrNtabCopyObjects.SpaceHost), + MrPushCopyObjects.SubscribedSearchTweet -> Set(MrNtabCopyObjects.SubscribedSearchTweet), + MrPushCopyObjects.TripGeoTweetPushCopy -> Set(MrNtabCopyObjects.RecommendedForYouCopy), + MrPushCopyObjects.Digest -> Set(MrNtabCopyObjects.Digest), + MrPushCopyObjects.TripHqTweetPushCopy -> Set(MrNtabCopyObjects.HighQualityTweet), + MrPushCopyObjects.ExploreVideoTweet -> Set(MrNtabCopyObjects.ExploreVideoTweet), + MrPushCopyObjects.ListRecommendation -> Set(MrNtabCopyObjects.ListRecommendation), + MrPushCopyObjects.MagicFanoutCreatorSubscription -> Set( + MrNtabCopyObjects.MagicFanoutCreatorSubscription), + MrPushCopyObjects.MagicFanoutNewCreator -> Set(MrNtabCopyObjects.MagicFanoutNewCreator) + ) + + /** + * + * @param crt - [[CommonRecommendationType]] used for a frigate push notification + * + * @return - Set of [[MRPushCopy]] objects representing push copies eligibile for a + * [[CommonRecommendationType]] + */ + def getPushCopiesFromRectype(crt: CommonRecommendationType): Option[Set[MRPushCopy]] = + rectypeToPushCopy.get(crt) + + /** + * + * @param pushcopy - [[MRPushCopy]] object representing a push notification copy + * @return - Set of [[MRNtabCopy]] objects that can be paired with a given [[MRPushCopy]] + */ + def getNtabcopiesFromPushcopy(pushcopy: MRPushCopy): Option[Set[MRNtabCopy]] = + pushcopyToNtabcopy.get(pushcopy) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.scala new file mode 100644 index 0000000000..0fe5f5cdd6 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.scala @@ -0,0 +1,41 @@ +package com.twitter.frigate.pushservice.refresh_handler.cross + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.util.MRPushCopy +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +private[cross] class CopyFilters(statsReceiver: StatsReceiver) { + + private val copyPredicates = new CopyPredicates(statsReceiver.scope("copy_predicate")) + + def execute(rawCandidate: RawCandidate, pushCopies: Seq[MRPushCopy]): Future[Seq[MRPushCopy]] = { + val candidateCopyPairs: Seq[CandidateCopyPair] = + pushCopies.map(CandidateCopyPair(rawCandidate, _)) + + val compositePredicate: Predicate[CandidateCopyPair] = rawCandidate match { + case _: F1FirstDegree | _: OutOfNetworkTweetCandidate | _: EventCandidate | + _: TopicProofTweetCandidate | _: ListPushCandidate | _: HermitInterestBasedUserFollow | + _: UserFollowWithoutSocialContextCandidate | _: DiscoverTwitterCandidate | + _: TopTweetImpressionsCandidate | _: TrendTweetCandidate | + _: SubscribedSearchTweetCandidate | _: DigestCandidate => + copyPredicates.alwaysTruePredicate + + case _: SocialContextActions => copyPredicates.displaySocialContextPredicate + + case _ => copyPredicates.unrecognizedCandidatePredicate // block unrecognised candidates + } + + // apply predicate to all [[MRPushCopy]] objects + val filterResults: Future[Seq[Boolean]] = compositePredicate(candidateCopyPairs) + filterResults.map { results: Seq[Boolean] => + val seqBuilder = Seq.newBuilder[MRPushCopy] + results.zip(pushCopies).foreach { + case (result, pushCopy) => if (result) seqBuilder += pushCopy + } + seqBuilder.result() + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.scala new file mode 100644 index 0000000000..980af15543 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.scala @@ -0,0 +1,36 @@ +package com.twitter.frigate.pushservice.refresh_handler.cross + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.SocialContextActions +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.hermit.predicate.Predicate + +class CopyPredicates(statsReceiver: StatsReceiver) { + val alwaysTruePredicate = Predicate + .from { _: CandidateCopyPair => + true + }.withStats(statsReceiver.scope("always_true_copy_predicate")) + + val unrecognizedCandidatePredicate = alwaysTruePredicate.flip + .withStats(statsReceiver.scope("unrecognized_candidate")) + + val displaySocialContextPredicate = Predicate + .from { candidateCopyPair: CandidateCopyPair => + candidateCopyPair.candidate match { + case candidateWithScActions: RawCandidate with SocialContextActions => + val socialContextUserIds = candidateWithScActions.socialContextActions.map(_.userId) + val countSocialContext = socialContextUserIds.size + val pushCopy = candidateCopyPair.pushCopy + + countSocialContext match { + case 1 => pushCopy.hasOneDisplaySocialContext && !pushCopy.hasOtherSocialContext + case 2 => pushCopy.hasTwoDisplayContext && !pushCopy.hasOtherSocialContext + case c if c > 2 => + pushCopy.hasOneDisplaySocialContext && pushCopy.hasOtherSocialContext + case _ => false + } + + case _ => false + } + }.withStats(statsReceiver.scope("display_social_context_predicate")) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.scala new file mode 100644 index 0000000000..90095056a6 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.scala @@ -0,0 +1,388 @@ +package com.twitter.frigate.pushservice.scriber + +import com.twitter.bijection.Base64String +import com.twitter.bijection.Injection +import com.twitter.bijection.scrooge.BinaryScalaCodec +import com.twitter.core_workflows.user_model.thriftscala.{UserState => ThriftUserState} +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.tracing.Trace +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.CandidateResult +import com.twitter.frigate.common.base.Invalid +import com.twitter.frigate.common.base.OK +import com.twitter.frigate.common.base.Result +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.data_pipeline.features_common.PushQualityModelFeatureContext +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.scribe.thriftscala.CandidateFilteredOutStep +import com.twitter.frigate.scribe.thriftscala.CandidateRequestInfo +import com.twitter.frigate.scribe.thriftscala.MrRequestScribe +import com.twitter.frigate.scribe.thriftscala.TargetUserInfo +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.frigate.thriftscala.TweetNotification +import com.twitter.frigate.thriftscala.{SocialContextAction => TSocialContextAction} +import com.twitter.logging.Logger +import com.twitter.ml.api.DataRecord +import com.twitter.ml.api.Feature +import com.twitter.ml.api.FeatureType +import com.twitter.ml.api.util.SRichDataRecord +import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions +import com.twitter.nrel.heavyranker.PushPredictionHelper +import com.twitter.util.Future +import com.twitter.util.Time +import java.util.UUID +import scala.collection.mutable + +class MrRequestScribeHandler(mrRequestScriberNode: String, stats: StatsReceiver) { + + private val mrRequestScribeLogger = Logger(mrRequestScriberNode) + + private val mrRequestScribeTargetFilteringStats = + stats.counter("MrRequestScribeHandler_target_filtering") + private val mrRequestScribeCandidateFilteringStats = + stats.counter("MrRequestScribeHandler_candidate_filtering") + private val mrRequestScribeInvalidStats = + stats.counter("MrRequestScribeHandler_invalid_filtering") + private val mrRequestScribeUnsupportedFeatureTypeStats = + stats.counter("MrRequestScribeHandler_unsupported_feature_type") + private val mrRequestScribeNotIncludedFeatureStats = + stats.counter("MrRequestScribeHandler_not_included_features") + + private final val MrRequestScribeInjection: Injection[MrRequestScribe, String] = BinaryScalaCodec( + MrRequestScribe + ) andThen Injection.connect[Array[Byte], Base64String, String] + + /** + * + * @param target : Target user id + * @param result : Result for target filtering + * + * @return + */ + def scribeForTargetFiltering(target: Target, result: Result): Future[Option[MrRequestScribe]] = { + if (target.isLoggedOutUser || !enableTargetFilteringScribing(target)) { + Future.None + } else { + val predicate = result match { + case Invalid(reason) => reason + case _ => + mrRequestScribeInvalidStats.incr() + throw new IllegalStateException("Invalid reason for Target Filtering " + result) + } + buildScribeThrift(target, predicate, None).map { targetFilteredScribe => + writeAtTargetFilteringStep(target, targetFilteredScribe) + Some(targetFilteredScribe) + } + } + } + + /** + * + * @param target : Target user id + * @param hydratedCandidates : Candidates hydrated with details: impressionId, frigateNotification and source + * @param preRankingFilteredCandidates : Candidates result filtered out at preRanking filtering step + * @param rankedCandidates : Sorted candidates details ranked by ranking step + * @param rerankedCandidates : Sorted candidates details ranked by reranking step + * @param restrictFilteredCandidates : Candidates details filtered out at restrict step + * @param allTakeCandidateResults : Candidates results at take step, include the candidates we take and the candidates filtered out at take step [with different result] + * + * @return + */ + def scribeForCandidateFiltering( + target: Target, + hydratedCandidates: Seq[CandidateDetails[PushCandidate]], + preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]], + rankedCandidates: Seq[CandidateDetails[PushCandidate]], + rerankedCandidates: Seq[CandidateDetails[PushCandidate]], + restrictFilteredCandidates: Seq[CandidateDetails[PushCandidate]], + allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]] + ): Future[Seq[MrRequestScribe]] = { + if (target.isLoggedOutUser || target.isEmailUser) { + Future.Nil + } else if (enableCandidateFilteringScribing(target)) { + val hydrateFeature = + target.params(PushFeatureSwitchParams.EnableMrRequestScribingWithFeatureHydrating) || + target.scribeFeatureForRequestScribe + + val candidateRequestInfoSeq = generateCandidatesScribeInfo( + hydratedCandidates, + preRankingFilteredCandidates, + rankedCandidates, + rerankedCandidates, + restrictFilteredCandidates, + allTakeCandidateResults, + isFeatureHydratingEnabled = hydrateFeature + ) + val flattenStructure = + target.params(PushFeatureSwitchParams.EnableFlattenMrRequestScribing) || hydrateFeature + candidateRequestInfoSeq.flatMap { candidateRequestInfos => + if (flattenStructure) { + Future.collect { + candidateRequestInfos.map { candidateRequestInfo => + buildScribeThrift(target, None, Some(Seq(candidateRequestInfo))) + .map { mrRequestScribe => + writeAtCandidateFilteringStep(target, mrRequestScribe) + mrRequestScribe + } + } + } + } else { + buildScribeThrift(target, None, Some(candidateRequestInfos)) + .map { mrRequestScribe => + writeAtCandidateFilteringStep(target, mrRequestScribe) + Seq(mrRequestScribe) + } + } + } + } else Future.Nil + + } + + private def buildScribeThrift( + target: Target, + targetFilteredOutPredicate: Option[String], + candidatesRequestInfo: Option[Seq[CandidateRequestInfo]] + ): Future[MrRequestScribe] = { + Future + .join( + target.targetUserState, + generateTargetFeatureScribeInfo(target), + target.targetUser).map { + case (userStateOption, targetFeatureOption, gizmoduckUserOpt) => + val userState = userStateOption.map(userState => ThriftUserState(userState.id)) + val targetFeatures = + targetFeatureOption.map(ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord) + val traceId = Trace.id.traceId.toLong + + MrRequestScribe( + requestId = UUID.randomUUID.toString.replaceAll("-", ""), + scribedTimeMs = Time.now.inMilliseconds, + targetUserId = target.targetId, + targetUserInfo = Some( + TargetUserInfo( + userState, + features = targetFeatures, + userType = gizmoduckUserOpt.map(_.userType)) + ), + targetFilteredOutPredicate = targetFilteredOutPredicate, + candidates = candidatesRequestInfo, + traceId = Some(traceId) + ) + } + } + + private def generateTargetFeatureScribeInfo( + target: Target + ): Future[Option[DataRecord]] = { + val featureList = + target.params(PushFeatureSwitchParams.TargetLevelFeatureListForMrRequestScribing) + if (featureList.nonEmpty) { + PushPredictionHelper + .getDataRecordFromTargetFeatureMap( + target.targetId, + target.featureMap, + stats + ).map { dataRecord => + val richRecord = + new SRichDataRecord(dataRecord, PushQualityModelFeatureContext.featureContext) + + val selectedRecord = + SRichDataRecord(new DataRecord(), PushQualityModelFeatureContext.featureContext) + featureList.map { featureName => + val feature: Feature[_] = { + try { + PushQualityModelFeatureContext.featureContext.getFeature(featureName) + } catch { + case _: Exception => + mrRequestScribeNotIncludedFeatureStats.incr() + throw new IllegalStateException( + "Scribing features not included in FeatureContext: " + featureName) + } + } + + richRecord.getFeatureValueOpt(feature).foreach { featureVal => + feature.getFeatureType() match { + case FeatureType.BINARY => + selectedRecord.setFeatureValue( + feature.asInstanceOf[Feature[Boolean]], + featureVal.asInstanceOf[Boolean]) + case FeatureType.CONTINUOUS => + selectedRecord.setFeatureValue( + feature.asInstanceOf[Feature[Double]], + featureVal.asInstanceOf[Double]) + case FeatureType.STRING => + selectedRecord.setFeatureValue( + feature.asInstanceOf[Feature[String]], + featureVal.asInstanceOf[String]) + case FeatureType.DISCRETE => + selectedRecord.setFeatureValue( + feature.asInstanceOf[Feature[Long]], + featureVal.asInstanceOf[Long]) + case _ => + mrRequestScribeUnsupportedFeatureTypeStats.incr() + } + } + } + Some(selectedRecord.getRecord) + } + } else Future.None + } + + private def generateCandidatesScribeInfo( + hydratedCandidates: Seq[CandidateDetails[PushCandidate]], + preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]], + rankedCandidates: Seq[CandidateDetails[PushCandidate]], + rerankedCandidates: Seq[CandidateDetails[PushCandidate]], + restrictFilteredCandidates: Seq[CandidateDetails[PushCandidate]], + allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]], + isFeatureHydratingEnabled: Boolean + ): Future[Seq[CandidateRequestInfo]] = { + val candidatesMap = new mutable.HashMap[String, CandidateRequestInfo] + + hydratedCandidates.foreach { hydratedCandidate => + val frgNotif = hydratedCandidate.candidate.frigateNotification + val simplifiedTweetNotificationOpt = frgNotif.tweetNotification.map { tweetNotification => + TweetNotification( + tweetNotification.tweetId, + Seq.empty[TSocialContextAction], + tweetNotification.tweetAuthorId) + } + val simplifiedFrigateNotification = FrigateNotification( + frgNotif.commonRecommendationType, + frgNotif.notificationDisplayLocation, + tweetNotification = simplifiedTweetNotificationOpt + ) + candidatesMap(hydratedCandidate.candidate.impressionId) = CandidateRequestInfo( + candidateId = "", + candidateSource = hydratedCandidate.source.substring( + 0, + Math.min(6, hydratedCandidate.source.length) + ), + frigateNotification = Some(simplifiedFrigateNotification), + modelScore = None, + rankPosition = None, + rerankPosition = None, + features = None, + isSent = Some(false) + ) + } + + preRankingFilteredCandidates.foreach { preRankingFilteredCandidateResult => + candidatesMap(preRankingFilteredCandidateResult.candidate.impressionId) = + candidatesMap(preRankingFilteredCandidateResult.candidate.impressionId) + .copy( + candidateFilteredOutPredicate = preRankingFilteredCandidateResult.result match { + case Invalid(reason) => reason + case _ => { + mrRequestScribeInvalidStats.incr() + throw new IllegalStateException( + "Invalid reason for Candidate Filtering " + preRankingFilteredCandidateResult.result) + } + }, + candidateFilteredOutStep = Some(CandidateFilteredOutStep.PreRankFiltering) + ) + } + + for { + _ <- Future.collectToTry { + rankedCandidates.zipWithIndex.map { + case (rankedCandidateDetail, index) => + val modelScoresFut = { + val crt = rankedCandidateDetail.candidate.commonRecType + if (RecTypes.notEligibleForModelScoreTracking.contains(crt)) Future.None + else rankedCandidateDetail.candidate.modelScores.map(Some(_)) + } + + modelScoresFut.map { modelScores => + candidatesMap(rankedCandidateDetail.candidate.impressionId) = + candidatesMap(rankedCandidateDetail.candidate.impressionId).copy( + rankPosition = Some(index), + modelScore = modelScores + ) + } + } + } + + _ = rerankedCandidates.zipWithIndex.foreach { + case (rerankedCandidateDetail, index) => { + candidatesMap(rerankedCandidateDetail.candidate.impressionId) = + candidatesMap(rerankedCandidateDetail.candidate.impressionId).copy( + rerankPosition = Some(index) + ) + } + } + + _ <- Future.collectToTry { + rerankedCandidates.map { rerankedCandidateDetail => + if (isFeatureHydratingEnabled) { + PushPredictionHelper + .getDataRecord( + rerankedCandidateDetail.candidate.target.targetHydrationContext, + rerankedCandidateDetail.candidate.target.featureMap, + rerankedCandidateDetail.candidate.candidateHydrationContext, + rerankedCandidateDetail.candidate.candidateFeatureMap(), + stats + ).map { features => + candidatesMap(rerankedCandidateDetail.candidate.impressionId) = + candidatesMap(rerankedCandidateDetail.candidate.impressionId).copy( + features = Some( + ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord(features)) + ) + } + } else Future.Unit + } + } + + _ = restrictFilteredCandidates.foreach { restrictFilteredCandidateDetatil => + candidatesMap(restrictFilteredCandidateDetatil.candidate.impressionId) = + candidatesMap(restrictFilteredCandidateDetatil.candidate.impressionId) + .copy(candidateFilteredOutStep = Some(CandidateFilteredOutStep.Restrict)) + } + + _ = allTakeCandidateResults.foreach { allTakeCandidateResult => + allTakeCandidateResult.result match { + case OK => + candidatesMap(allTakeCandidateResult.candidate.impressionId) = + candidatesMap(allTakeCandidateResult.candidate.impressionId).copy(isSent = Some(true)) + case Invalid(reason) => + candidatesMap(allTakeCandidateResult.candidate.impressionId) = + candidatesMap(allTakeCandidateResult.candidate.impressionId).copy( + candidateFilteredOutPredicate = reason, + candidateFilteredOutStep = Some(CandidateFilteredOutStep.PostRankFiltering)) + case _ => + mrRequestScribeInvalidStats.incr() + throw new IllegalStateException( + "Invalid reason for Candidate Filtering " + allTakeCandidateResult.result) + } + } + } yield candidatesMap.values.toSeq + } + + private def enableTargetFilteringScribing(target: Target): Boolean = { + target.params(PushParams.EnableMrRequestScribing) && target.params( + PushFeatureSwitchParams.EnableMrRequestScribingForTargetFiltering) + } + + private def enableCandidateFilteringScribing(target: Target): Boolean = { + target.params(PushParams.EnableMrRequestScribing) && target.params( + PushFeatureSwitchParams.EnableMrRequestScribingForCandidateFiltering) + } + + private def writeAtTargetFilteringStep(target: Target, mrRequestScribe: MrRequestScribe) = { + logToScribe(mrRequestScribe) + mrRequestScribeTargetFilteringStats.incr() + } + + private def writeAtCandidateFilteringStep(target: Target, mrRequestScribe: MrRequestScribe) = { + logToScribe(mrRequestScribe) + mrRequestScribeCandidateFilteringStats.incr() + } + + private def logToScribe(mrRequestScribe: MrRequestScribe): Unit = { + val logEntry: String = MrRequestScribeInjection(mrRequestScribe) + mrRequestScribeLogger.info(logEntry) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.scala new file mode 100644 index 0000000000..e235f76cbc --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.scala @@ -0,0 +1,250 @@ +package com.twitter.frigate.pushservice.send_handler + +import com.twitter.finagle.stats.BroadcastStatsReceiver +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.CandidateFilteringOnlyFlow +import com.twitter.frigate.common.base.CandidateResult +import com.twitter.frigate.common.base.FeatureMap +import com.twitter.frigate.common.base.OK +import com.twitter.frigate.common.base.Response +import com.twitter.frigate.common.base.Result +import com.twitter.frigate.common.base.Stats.track +import com.twitter.frigate.common.config.CommonConstants +import com.twitter.frigate.common.logger.MRLogger +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.common.util.InvalidRequestException +import com.twitter.frigate.common.util.MrNtabCopyObjects +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.ml.HydrationContextBuilder +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.EnableMagicFanoutNewsForYouNtabCopy +import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler +import com.twitter.frigate.pushservice.send_handler.generator.PushRequestToCandidate +import com.twitter.frigate.pushservice.take.SendHandlerNotifier +import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPostCandidateValidator +import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPreCandidateValidator +import com.twitter.frigate.pushservice.target.PushTargetUserBuilder +import com.twitter.frigate.pushservice.util.ResponseStatsTrackUtils.trackStatsForResponseToRequest +import com.twitter.frigate.pushservice.util.SendHandlerPredicateUtil +import com.twitter.frigate.pushservice.thriftscala.PushRequest +import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe +import com.twitter.frigate.pushservice.thriftscala.PushResponse +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.nrel.heavyranker.FeatureHydrator +import com.twitter.util._ + +/** + * A handler for sending PushRequests + */ +class SendHandler( + pushTargetUserBuilder: PushTargetUserBuilder, + preCandidateValidator: SendHandlerPreCandidateValidator, + postCandidateValidator: SendHandlerPostCandidateValidator, + sendHandlerNotifier: SendHandlerNotifier, + candidateHydrator: SendHandlerPushCandidateHydrator, + featureHydrator: FeatureHydrator, + sendHandlerPredicateUtil: SendHandlerPredicateUtil, + mrRequestScriberNode: String +)( + implicit val statsReceiver: StatsReceiver, + implicit val config: Config) + extends CandidateFilteringOnlyFlow[Target, RawCandidate, PushCandidate] { + + implicit private val timer: Timer = new JavaTimer(true) + val stats = statsReceiver.scope("SendHandler") + val log = MRLogger("SendHandler") + + private val buildTargetStats = stats.scope("build_target") + + private val candidateHydrationLatency: Stat = + stats.stat("candidateHydrationLatency") + + private val candidatePreValidatorLatency: Stat = + stats.stat("candidatePreValidatorLatency") + + private val candidatePostValidatorLatency: Stat = + stats.stat("candidatePostValidatorLatency") + + private val featureHydrationLatency: StatsReceiver = + stats.scope("featureHydrationLatency") + + private val mrRequestScribeHandler = + new MrRequestScribeHandler(mrRequestScriberNode, stats.scope("mr_request_scribe")) + + def apply(request: PushRequest): Future[PushResponse] = { + val receivers = Seq( + stats, + stats.scope(request.notification.commonRecommendationType.toString) + ) + val bStats = BroadcastStatsReceiver(receivers) + bStats.counter("requests").incr() + Stat + .timeFuture(bStats.stat("latency"))( + process(request).raiseWithin(CommonConstants.maxPushRequestDuration)) + .onSuccess { + case (pushResp, rawCandidate) => + trackStatsForResponseToRequest( + rawCandidate.commonRecType, + rawCandidate.target, + pushResp, + receivers)(statsReceiver) + if (!request.context.exists(_.darkWrite.contains(true))) { + config.requestScribe(PushRequestScribe(request, pushResp)) + } + } + .onFailure { ex => + bStats.counter("failures").incr() + bStats.scope("failures").counter(ex.getClass.getCanonicalName).incr() + } + .map { + case (pushResp, _) => pushResp + } + } + + private def process(request: PushRequest): Future[(PushResponse, RawCandidate)] = { + val recType = request.notification.commonRecommendationType + + track(buildTargetStats)( + pushTargetUserBuilder + .buildTarget( + request.userId, + request.context + ) + ).flatMap { targetUser => + val responseWithScribedInfo = request.context.exists { context => + context.responseWithScribedInfo.contains(true) + } + val newRequest = + if (request.notification.commonRecommendationType == CommonRecommendationType.MagicFanoutNewsEvent && + targetUser.params(EnableMagicFanoutNewsForYouNtabCopy)) { + val newNotification = request.notification.copy(ntabCopyId = + Some(MrNtabCopyObjects.MagicFanoutNewsForYouCopy.copyId)) + request.copy(notification = newNotification) + } else request + + if (RecTypes.isSendHandlerType(recType) || newRequest.context.exists( + _.allowCRT.contains(true))) { + + val rawCandidateFut = PushRequestToCandidate.generatePushCandidate( + newRequest.notification, + targetUser + ) + + rawCandidateFut.flatMap { rawCandidate => + val pushResponse = process(targetUser, Seq(rawCandidate)).flatMap { + sendHandlerNotifier.checkResponseAndNotify(_, responseWithScribedInfo) + } + + pushResponse.map { pushResponse => + (pushResponse, rawCandidate) + } + } + } else { + Future.exception(InvalidRequestException(s"${recType.name} not supported in SendHandler")) + } + } + } + + private def hydrateFeatures( + candidateDetails: Seq[CandidateDetails[PushCandidate]], + target: Target, + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + + candidateDetails.headOption match { + case Some(candidateDetail) + if RecTypes.notEligibleForModelScoreTracking(candidateDetail.candidate.commonRecType) => + Future.value(candidateDetails) + + case Some(candidateDetail) => + val hydrationContextFut = HydrationContextBuilder.build(candidateDetail.candidate) + hydrationContextFut.flatMap { hc => + featureHydrator + .hydrateCandidate(Seq(hc), target.mrRequestContextForFeatureStore) + .map { hydrationResult => + val features = hydrationResult.getOrElse(hc, FeatureMap()) + candidateDetail.candidate.mergeFeatures(features) + candidateDetails + } + } + case _ => Future.Nil + } + } + + override def process( + target: Target, + externalCandidates: Seq[RawCandidate] + ): Future[Response[PushCandidate, Result]] = { + val candidate = externalCandidates.map(CandidateDetails(_, "realtime")) + + for { + hydratedCandidatesWithCopy <- hydrateCandidates(candidate) + + (candidates, preHydrationFilteredCandidates) <- track(filterStats)( + filter(target, hydratedCandidatesWithCopy) + ) + + featureHydratedCandidates <- + track(featureHydrationLatency)(hydrateFeatures(candidates, target)) + + allTakeCandidateResults <- track(takeStats)( + take(target, featureHydratedCandidates, desiredCandidateCount(target)) + ) + + _ <- mrRequestScribeHandler.scribeForCandidateFiltering( + target = target, + hydratedCandidates = hydratedCandidatesWithCopy, + preRankingFilteredCandidates = preHydrationFilteredCandidates, + rankedCandidates = featureHydratedCandidates, + rerankedCandidates = Seq.empty, + restrictFilteredCandidates = Seq.empty, // no restrict step + allTakeCandidateResults = allTakeCandidateResults + ) + } yield { + + /** + * We combine the results for all filtering steps and pass on in sequence to next step + * + * This is done to ensure the filtering reason for the candidate from multiple levels of + * filtering is carried all the way until [[PushResponse]] is built and returned from + * frigate-pushservice-send + */ + Response(OK, allTakeCandidateResults ++ preHydrationFilteredCandidates) + } + } + + override def hydrateCandidates( + candidates: Seq[CandidateDetails[RawCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + Stat.timeFuture(candidateHydrationLatency)(candidateHydrator(candidates)) + } + + // Filter Step - pre-predicates and app specific predicates + override def filter( + target: Target, + hydratedCandidatesDetails: Seq[CandidateDetails[PushCandidate]] + ): Future[ + (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) + ] = { + Stat.timeFuture(candidatePreValidatorLatency)( + sendHandlerPredicateUtil.preValidationForCandidate( + hydratedCandidatesDetails, + preCandidateValidator + )) + } + + // Post Validation - Take step + override def validCandidates( + target: Target, + candidates: Seq[PushCandidate] + ): Future[Seq[Result]] = { + Stat.timeFuture(candidatePostValidatorLatency)(Future.collect(candidates.map { candidate => + sendHandlerPredicateUtil + .postValidationForCandidate(candidate, postCandidateValidator) + .map(res => res.result) + })) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.scala new file mode 100644 index 0000000000..f8f1027901 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.scala @@ -0,0 +1,184 @@ +package com.twitter.frigate.pushservice.send_handler + +import com.twitter.escherbird.metadata.thriftscala.EntityMegadata +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext +import com.twitter.frigate.common.util.MrNtabCopyObjects +import com.twitter.frigate.common.util.MrPushCopyObjects +import com.twitter.frigate.magic_events.thriftscala.FanoutEvent +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.store.EventRequest +import com.twitter.frigate.pushservice.store.UttEntityHydrationStore +import com.twitter.frigate.pushservice.util.CandidateHydrationUtil._ +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery +import com.twitter.interests.thriftscala.UserInterests +import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} +import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities +import com.twitter.storehaus.ReadableStore +import com.twitter.strato.client.UserId +import com.twitter.ubs.thriftscala.AudioSpace +import com.twitter.util.Future + +case class SendHandlerPushCandidateHydrator( + lexServiceStore: ReadableStore[EventRequest, LiveEvent], + fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], + semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], + safeUserStore: ReadableStore[Long, User], + simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities], + audioSpaceStore: ReadableStore[String, AudioSpace], + interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], + uttEntityHydrationStore: UttEntityHydrationStore, + superFollowCreatorTweetCountStore: ReadableStore[UserId, Int] +)( + implicit statsReceiver: StatsReceiver, + implicit val weightedOpenOrNtabClickModelScorer: PushMLModelScorer) { + + lazy val candidateWithCopyNumStat = statsReceiver.stat("candidate_with_copy_num") + lazy val hydratedCandidateStat = statsReceiver.scope("hydrated_candidates") + + def updateCandidates( + candidateDetails: Seq[CandidateDetails[RawCandidate]], + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + + Future.collect { + candidateDetails.map { candidateDetail => + val pushCandidate = candidateDetail.candidate + + val copyIds = getCopyIdsByCRT(pushCandidate.commonRecType) + + val hydratedCandidateFut = pushCandidate match { + case magicFanoutNewsEventCandidate: MagicFanoutNewsEventCandidate => + getHydratedCandidateForMagicFanoutNewsEvent( + magicFanoutNewsEventCandidate, + copyIds, + lexServiceStore, + fanoutMetadataStore, + semanticCoreMegadataStore, + simClusterToEntityStore, + interestsLookupStore, + uttEntityHydrationStore + ) + + case scheduledSpaceSubscriberCandidate: ScheduledSpaceSubscriberCandidate => + getHydratedCandidateForScheduledSpaceSubscriber( + scheduledSpaceSubscriberCandidate, + safeUserStore, + copyIds, + audioSpaceStore + ) + case scheduledSpaceSpeakerCandidate: ScheduledSpaceSpeakerCandidate => + getHydratedCandidateForScheduledSpaceSpeaker( + scheduledSpaceSpeakerCandidate, + safeUserStore, + copyIds, + audioSpaceStore + ) + case magicFanoutSportsEventCandidate: MagicFanoutSportsEventCandidate with MagicFanoutSportsScoreInformation => + getHydratedCandidateForMagicFanoutSportsEvent( + magicFanoutSportsEventCandidate, + copyIds, + lexServiceStore, + fanoutMetadataStore, + semanticCoreMegadataStore, + interestsLookupStore, + uttEntityHydrationStore + ) + case magicFanoutProductLaunchCandidate: MagicFanoutProductLaunchCandidate => + getHydratedCandidateForMagicFanoutProductLaunch( + magicFanoutProductLaunchCandidate, + copyIds) + case creatorEventCandidate: MagicFanoutCreatorEventCandidate => + getHydratedCandidateForMagicFanoutCreatorEvent( + creatorEventCandidate, + safeUserStore, + copyIds, + superFollowCreatorTweetCountStore) + case _ => + throw new IllegalArgumentException("Incorrect candidate type when update candidates") + } + + hydratedCandidateFut.map { hydratedCandidate => + hydratedCandidateStat.counter(hydratedCandidate.commonRecType.name).incr() + CandidateDetails( + hydratedCandidate, + source = candidateDetail.source + ) + } + } + } + } + + private def getCopyIdsByCRT(crt: CommonRecommendationType): CopyIds = { + crt match { + case CommonRecommendationType.MagicFanoutNewsEvent => + CopyIds( + pushCopyId = Some(MrPushCopyObjects.MagicFanoutNewsPushCopy.copyId), + ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutNewsForYouCopy.copyId), + aggregationId = None + ) + + case CommonRecommendationType.ScheduledSpaceSubscriber => + CopyIds( + pushCopyId = Some(MrPushCopyObjects.ScheduledSpaceSubscriber.copyId), + ntabCopyId = Some(MrNtabCopyObjects.ScheduledSpaceSubscriber.copyId), + aggregationId = None + ) + case CommonRecommendationType.ScheduledSpaceSpeaker => + CopyIds( + pushCopyId = Some(MrPushCopyObjects.ScheduledSpaceSpeaker.copyId), + ntabCopyId = Some(MrNtabCopyObjects.ScheduledSpaceSpeakerNow.copyId), + aggregationId = None + ) + case CommonRecommendationType.SpaceSpeaker => + CopyIds( + pushCopyId = Some(MrPushCopyObjects.SpaceSpeaker.copyId), + ntabCopyId = Some(MrNtabCopyObjects.SpaceSpeaker.copyId), + aggregationId = None + ) + case CommonRecommendationType.SpaceHost => + CopyIds( + pushCopyId = Some(MrPushCopyObjects.SpaceHost.copyId), + ntabCopyId = Some(MrNtabCopyObjects.SpaceHost.copyId), + aggregationId = None + ) + case CommonRecommendationType.MagicFanoutSportsEvent => + CopyIds( + pushCopyId = Some(MrPushCopyObjects.MagicFanoutSportsPushCopy.copyId), + ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutSportsCopy.copyId), + aggregationId = None + ) + case CommonRecommendationType.MagicFanoutProductLaunch => + CopyIds( + pushCopyId = Some(MrPushCopyObjects.MagicFanoutProductLaunch.copyId), + ntabCopyId = Some(MrNtabCopyObjects.ProductLaunch.copyId), + aggregationId = None + ) + case CommonRecommendationType.CreatorSubscriber => + CopyIds( + pushCopyId = Some(MrPushCopyObjects.MagicFanoutCreatorSubscription.copyId), + ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutCreatorSubscription.copyId), + aggregationId = None + ) + case CommonRecommendationType.NewCreator => + CopyIds( + pushCopyId = Some(MrPushCopyObjects.MagicFanoutNewCreator.copyId), + ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutNewCreator.copyId), + aggregationId = None + ) + case _ => + throw new IllegalArgumentException("Incorrect candidate type when fetch copy ids") + } + } + + def apply( + candidateDetails: Seq[CandidateDetails[RawCandidate]] + ): Future[Seq[CandidateDetails[PushCandidate]]] = { + updateCandidates(candidateDetails) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.scala new file mode 100644 index 0000000000..45907fa8e9 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.scala @@ -0,0 +1,17 @@ +package com.twitter.frigate.pushservice.send_handler.generator + +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.util.Future + +trait CandidateGenerator { + + /** + * Build RawCandidate from FrigateNotification + * @param target + * @param frigateNotification + * @return RawCandidate + */ + def getCandidate(target: Target, frigateNotification: FrigateNotification): Future[RawCandidate] +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.scala new file mode 100644 index 0000000000..10a7acb896 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.scala @@ -0,0 +1,70 @@ +package com.twitter.frigate.pushservice.send_handler.generator + +import com.twitter.frigate.common.base.MagicFanoutCreatorEventCandidate +import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType +import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.model.PushTypes +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.util.Future + +object MagicFanoutCreatorEventCandidateGenerator extends CandidateGenerator { + override def getCandidate( + targetUser: PushTypes.Target, + notification: FrigateNotification + ): Future[PushTypes.RawCandidate] = { + + require( + notification.commonRecommendationType == CommonRecommendationType.CreatorSubscriber || notification.commonRecommendationType == CommonRecommendationType.NewCreator, + "MagicFanoutCreatorEvent: unexpected CRT " + notification.commonRecommendationType + ) + require( + notification.creatorSubscriptionNotification.isDefined, + "MagicFanoutCreatorEvent: creatorSubscriptionNotification is not defined") + require( + notification.creatorSubscriptionNotification.exists(_.magicFanoutPushId.isDefined), + "MagicFanoutCreatorEvent: magicFanoutPushId is not defined") + require( + notification.creatorSubscriptionNotification.exists(_.fanoutReasons.isDefined), + "MagicFanoutCreatorEvent: fanoutReasons is not defined") + require( + notification.creatorSubscriptionNotification.exists(_.creatorId.isDefined), + "MagicFanoutCreatorEvent: creatorId is not defined") + if (notification.commonRecommendationType == CommonRecommendationType.CreatorSubscriber) { + require( + notification.creatorSubscriptionNotification + .exists(_.subscriberId.isDefined), + "MagicFanoutCreatorEvent: subscriber id is not defined" + ) + } + + val creatorSubscriptionNotification = notification.creatorSubscriptionNotification.get + + val candidate = new RawCandidate with MagicFanoutCreatorEventCandidate { + + override val target: Target = targetUser + + override val pushId: Long = + creatorSubscriptionNotification.magicFanoutPushId.get + + override val candidateMagicEventsReasons: Seq[MagicEventsReason] = + creatorSubscriptionNotification.fanoutReasons.get + + override val creatorFanoutType: CreatorFanoutType = + creatorSubscriptionNotification.creatorFanoutType + + override val commonRecType: CommonRecommendationType = + notification.commonRecommendationType + + override val frigateNotification: FrigateNotification = notification + + override val subscriberId: Option[Long] = creatorSubscriptionNotification.subscriberId + + override val creatorId: Long = creatorSubscriptionNotification.creatorId.get + } + + Future.value(candidate) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.scala new file mode 100644 index 0000000000..7b351c91a5 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.scala @@ -0,0 +1,57 @@ +package com.twitter.frigate.pushservice.send_handler.generator + +import com.twitter.frigate.common.base.MagicFanoutNewsEventCandidate +import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails +import com.twitter.util.Future + +object MagicFanoutNewsEventCandidateGenerator extends CandidateGenerator { + + override def getCandidate( + targetUser: Target, + notification: FrigateNotification + ): Future[RawCandidate] = { + + /** + * frigateNotification recommendation type should be [[CommonRecommendationType.MagicFanoutNewsEvent]] + * AND pushId field should be set + **/ + require( + notification.commonRecommendationType == CommonRecommendationType.MagicFanoutNewsEvent, + "MagicFanoutNewsEvent: unexpected CRT " + notification.commonRecommendationType + ) + + require( + notification.magicFanoutEventNotification.exists(_.pushId.isDefined), + "MagicFanoutNewsEvent: pushId is not defined") + + val magicFanoutEventNotification = notification.magicFanoutEventNotification.get + + val candidate = new RawCandidate with MagicFanoutNewsEventCandidate { + + override val target: Target = targetUser + + override val eventId: Long = magicFanoutEventNotification.eventId + + override val pushId: Long = magicFanoutEventNotification.pushId.get + + override val candidateMagicEventsReasons: Seq[MagicEventsReason] = + magicFanoutEventNotification.eventReasons.getOrElse(Seq.empty) + + override val momentId: Option[Long] = magicFanoutEventNotification.momentId + + override val eventLanguage: Option[String] = magicFanoutEventNotification.eventLanguage + + override val details: Option[MagicFanoutEventNotificationDetails] = + magicFanoutEventNotification.details + + override val frigateNotification: FrigateNotification = notification + } + + Future.value(candidate) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.scala new file mode 100644 index 0000000000..6844b1b06c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.scala @@ -0,0 +1,54 @@ +package com.twitter.frigate.pushservice.send_handler.generator + +import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate +import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason +import com.twitter.frigate.magic_events.thriftscala.ProductType +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.model.PushTypes +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.util.Future + +object MagicFanoutProductLaunchCandidateGenerator extends CandidateGenerator { + + override def getCandidate( + targetUser: PushTypes.Target, + notification: FrigateNotification + ): Future[PushTypes.RawCandidate] = { + + require( + notification.commonRecommendationType == CommonRecommendationType.MagicFanoutProductLaunch, + "MagicFanoutProductLaunch: unexpected CRT " + notification.commonRecommendationType + ) + require( + notification.magicFanoutProductLaunchNotification.isDefined, + "MagicFanoutProductLaunch: magicFanoutProductLaunchNotification is not defined") + require( + notification.magicFanoutProductLaunchNotification.exists(_.magicFanoutPushId.isDefined), + "MagicFanoutProductLaunch: magicFanoutPushId is not defined") + require( + notification.magicFanoutProductLaunchNotification.exists(_.fanoutReasons.isDefined), + "MagicFanoutProductLaunch: fanoutReasons is not defined") + + val magicFanoutProductLaunchNotification = notification.magicFanoutProductLaunchNotification.get + + val candidate = new RawCandidate with MagicFanoutProductLaunchCandidate { + + override val target: Target = targetUser + + override val pushId: Long = + magicFanoutProductLaunchNotification.magicFanoutPushId.get + + override val candidateMagicEventsReasons: Seq[MagicEventsReason] = + magicFanoutProductLaunchNotification.fanoutReasons.get + + override val productLaunchType: ProductType = + magicFanoutProductLaunchNotification.productLaunchType + + override val frigateNotification: FrigateNotification = notification + } + + Future.value(candidate) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.scala new file mode 100644 index 0000000000..cdd37833ed --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.scala @@ -0,0 +1,153 @@ +package com.twitter.frigate.pushservice.send_handler.generator + +import com.twitter.datatools.entityservice.entities.sports.thriftscala.BaseballGameLiveUpdate +import com.twitter.datatools.entityservice.entities.sports.thriftscala.BasketballGameLiveUpdate +import com.twitter.datatools.entityservice.entities.sports.thriftscala.CricketMatchLiveUpdate +import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate +import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate +import com.twitter.escherbird.common.thriftscala.Domains +import com.twitter.escherbird.common.thriftscala.QualifiedId +import com.twitter.escherbird.metadata.thriftscala.EntityMegadata +import com.twitter.frigate.common.base.BaseGameScore +import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate +import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation +import com.twitter.frigate.common.base.TeamInfo +import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason +import com.twitter.frigate.pushservice.exception.InvalidSportDomainException +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutSportsUtil +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails +import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +object MagicFanoutSportsEventCandidateGenerator { + + final def getCandidate( + targetUser: Target, + notification: FrigateNotification, + basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate], + baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate], + cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate], + soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate], + nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate], + semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], + ): Future[RawCandidate] = { + + /** + * frigateNotification recommendation type should be [[CommonRecommendationType.MagicFanoutSportsEvent]] + * AND pushId field should be set + * + * */ + require( + notification.commonRecommendationType == CommonRecommendationType.MagicFanoutSportsEvent, + "MagicFanoutSports: unexpected CRT " + notification.commonRecommendationType + ) + + require( + notification.magicFanoutEventNotification.exists(_.pushId.isDefined), + "MagicFanoutSportsEvent: pushId is not defined") + + val magicFanoutEventNotification = notification.magicFanoutEventNotification.get + val eventId = magicFanoutEventNotification.eventId + val _isScoreUpdate = magicFanoutEventNotification.isScoreUpdate.getOrElse(false) + + val gameScoresFut: Future[Option[BaseGameScore]] = { + if (_isScoreUpdate) { + semanticCoreMegadataStore + .get(SemanticEntityForQuery(PushConstants.SportsEventDomainId, eventId)) + .flatMap { + case Some(megadata) => + if (megadata.domains.contains(Domains.BasketballGame)) { + basketballGameScoreStore + .get(QualifiedId(Domains.BasketballGame.value, eventId)).map { + case Some(game) if game.status.isDefined => + val status = game.status.get + MagicFanoutSportsUtil.transformToGameScore(game.score, status) + case _ => None + } + } else if (megadata.domains.contains(Domains.BaseballGame)) { + baseballGameScoreStore + .get(QualifiedId(Domains.BaseballGame.value, eventId)).map { + case Some(game) if game.status.isDefined => + val status = game.status.get + MagicFanoutSportsUtil.transformToGameScore(game.runs, status) + case _ => None + } + } else if (megadata.domains.contains(Domains.NflFootballGame)) { + nflGameScoreStore + .get(QualifiedId(Domains.NflFootballGame.value, eventId)).map { + case Some(game) if game.status.isDefined => + val nflScore = MagicFanoutSportsUtil.transformNFLGameScore(game) + nflScore + case _ => None + } + } else if (megadata.domains.contains(Domains.SoccerMatch)) { + soccerMatchScoreStore + .get(QualifiedId(Domains.SoccerMatch.value, eventId)).map { + case Some(game) if game.status.isDefined => + val soccerScore = MagicFanoutSportsUtil.transformSoccerGameScore(game) + soccerScore + case _ => None + } + } else { + // The domains are not in our list of supported sports + throw new InvalidSportDomainException( + s"Domain for entity ${eventId} is not supported") + } + case _ => Future.None + } + } else Future.None + } + + val homeTeamInfoFut: Future[Option[TeamInfo]] = gameScoresFut.flatMap { + case Some(gameScore) => + MagicFanoutSportsUtil.getTeamInfo(gameScore.home, semanticCoreMegadataStore) + case _ => Future.None + } + + val awayTeamInfoFut: Future[Option[TeamInfo]] = gameScoresFut.flatMap { + case Some(gameScore) => + MagicFanoutSportsUtil.getTeamInfo(gameScore.away, semanticCoreMegadataStore) + case _ => Future.None + } + + val candidate = new RawCandidate + with MagicFanoutSportsEventCandidate + with MagicFanoutSportsScoreInformation { + + override val target: Target = targetUser + + override val eventId: Long = magicFanoutEventNotification.eventId + + override val pushId: Long = magicFanoutEventNotification.pushId.get + + override val candidateMagicEventsReasons: Seq[MagicEventsReason] = + magicFanoutEventNotification.eventReasons.getOrElse(Seq.empty) + + override val momentId: Option[Long] = magicFanoutEventNotification.momentId + + override val eventLanguage: Option[String] = magicFanoutEventNotification.eventLanguage + + override val details: Option[MagicFanoutEventNotificationDetails] = + magicFanoutEventNotification.details + + override val frigateNotification: FrigateNotification = notification + + override val homeTeamInfo: Future[Option[TeamInfo]] = homeTeamInfoFut + + override val awayTeamInfo: Future[Option[TeamInfo]] = awayTeamInfoFut + + override val gameScores: Future[Option[BaseGameScore]] = gameScoresFut + + override val isScoreUpdate: Boolean = _isScoreUpdate + } + + Future.value(candidate) + + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.scala new file mode 100644 index 0000000000..8d7e81d3f8 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.scala @@ -0,0 +1,49 @@ +package com.twitter.frigate.pushservice.send_handler.generator + +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.exception.UnsupportedCrtException +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} +import com.twitter.util.Future + +object PushRequestToCandidate { + final def generatePushCandidate( + frigateNotification: FrigateNotification, + target: Target + )( + implicit config: Config + ): Future[RawCandidate] = { + + val candidateGenerator: (Target, FrigateNotification) => Future[RawCandidate] = { + frigateNotification.commonRecommendationType match { + case CRT.MagicFanoutNewsEvent => MagicFanoutNewsEventCandidateGenerator.getCandidate + case CRT.ScheduledSpaceSubscriber => ScheduledSpaceSubscriberCandidateGenerator.getCandidate + case CRT.ScheduledSpaceSpeaker => ScheduledSpaceSpeakerCandidateGenerator.getCandidate + case CRT.MagicFanoutSportsEvent => + MagicFanoutSportsEventCandidateGenerator.getCandidate( + _, + _, + config.basketballGameScoreStore, + config.baseballGameScoreStore, + config.cricketMatchScoreStore, + config.soccerMatchScoreStore, + config.nflGameScoreStore, + config.semanticCoreMegadataStore + ) + case CRT.MagicFanoutProductLaunch => + MagicFanoutProductLaunchCandidateGenerator.getCandidate + case CRT.NewCreator => + MagicFanoutCreatorEventCandidateGenerator.getCandidate + case CRT.CreatorSubscriber => + MagicFanoutCreatorEventCandidateGenerator.getCandidate + case _ => + throw new UnsupportedCrtException( + "UnsupportedCrtException for SendHandler: " + frigateNotification.commonRecommendationType) + } + } + + candidateGenerator(target, frigateNotification) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.scala new file mode 100644 index 0000000000..e7821db76f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.scala @@ -0,0 +1,55 @@ +package com.twitter.frigate.pushservice.send_handler.generator + +import com.twitter.frigate.common.base.ScheduledSpaceSpeakerCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.util.Future + +object ScheduledSpaceSpeakerCandidateGenerator extends CandidateGenerator { + + override def getCandidate( + targetUser: Target, + notification: FrigateNotification + ): Future[RawCandidate] = { + + /** + * frigateNotification recommendation type should be [[CommonRecommendationType.ScheduledSpaceSpeaker]] + * + **/ + require( + notification.commonRecommendationType == CommonRecommendationType.ScheduledSpaceSpeaker, + "ScheduledSpaceSpeaker: unexpected CRT " + notification.commonRecommendationType + ) + + val spaceNotification = notification.spaceNotification.getOrElse( + throw new IllegalStateException("ScheduledSpaceSpeaker notification object not defined")) + + require( + spaceNotification.hostUserId.isDefined, + "ScheduledSpaceSpeaker notification - hostUserId not defined" + ) + + val spaceHostId = spaceNotification.hostUserId + + require( + spaceNotification.scheduledStartTime.isDefined, + "ScheduledSpaceSpeaker notification - scheduledStartTime not defined" + ) + + val scheduledStartTime = spaceNotification.scheduledStartTime.get + + val candidate = new RawCandidate with ScheduledSpaceSpeakerCandidate { + override val target: Target = targetUser + override val frigateNotification: FrigateNotification = notification + override val spaceId: String = spaceNotification.broadcastId + override val hostId: Option[Long] = spaceHostId + override val startTime: Long = scheduledStartTime + override val speakerIds: Option[Seq[Long]] = spaceNotification.speakers + override val listenerIds: Option[Seq[Long]] = spaceNotification.listeners + } + + Future.value(candidate) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.scala new file mode 100644 index 0000000000..484f17b2a1 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.scala @@ -0,0 +1,55 @@ +package com.twitter.frigate.pushservice.send_handler.generator + +import com.twitter.frigate.common.base.ScheduledSpaceSubscriberCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.util.Future + +object ScheduledSpaceSubscriberCandidateGenerator extends CandidateGenerator { + + override def getCandidate( + targetUser: Target, + notification: FrigateNotification + ): Future[RawCandidate] = { + + /** + * frigateNotification recommendation type should be [[CommonRecommendationType.ScheduledSpaceSubscriber]] + * + **/ + require( + notification.commonRecommendationType == CommonRecommendationType.ScheduledSpaceSubscriber, + "ScheduledSpaceSubscriber: unexpected CRT " + notification.commonRecommendationType + ) + + val spaceNotification = notification.spaceNotification.getOrElse( + throw new IllegalStateException("ScheduledSpaceSubscriber notification object not defined")) + + require( + spaceNotification.hostUserId.isDefined, + "ScheduledSpaceSubscriber notification - hostUserId not defined" + ) + + val spaceHostId = spaceNotification.hostUserId + + require( + spaceNotification.scheduledStartTime.isDefined, + "ScheduledSpaceSubscriber notification - scheduledStartTime not defined" + ) + + val scheduledStartTime = spaceNotification.scheduledStartTime.get + + val candidate = new RawCandidate with ScheduledSpaceSubscriberCandidate { + override val target: Target = targetUser + override val frigateNotification: FrigateNotification = notification + override val spaceId: String = spaceNotification.broadcastId + override val hostId: Option[Long] = spaceHostId + override val startTime: Long = scheduledStartTime + override val speakerIds: Option[Seq[Long]] = spaceNotification.speakers + override val listenerIds: Option[Seq[Long]] = spaceNotification.listeners + } + + Future.value(candidate) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.scala new file mode 100644 index 0000000000..1f4171030b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.scala @@ -0,0 +1,17 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.content_mixer.thriftscala.ContentMixer +import com.twitter.content_mixer.thriftscala.ContentMixerRequest +import com.twitter.content_mixer.thriftscala.ContentMixerResponse +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +case class ContentMixerStore(contentMixer: ContentMixer.MethodPerEndpoint) + extends ReadableStore[ContentMixerRequest, ContentMixerResponse] { + + override def get(request: ContentMixerRequest): Future[Option[ContentMixerResponse]] = { + contentMixer.getCandidates(request).map { response => + Some(response) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.scala new file mode 100644 index 0000000000..b793ade7e3 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.scala @@ -0,0 +1,15 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.copyselectionservice.thriftscala._ +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +class CopySelectionServiceStore(copySelectionServiceClient: CopySelectionService.FinagledClient) + extends ReadableStore[CopySelectionRequestV1, Copy] { + override def get(k: CopySelectionRequestV1): Future[Option[Copy]] = + copySelectionServiceClient.getSelectedCopy(CopySelectionRequest.V1(k)).map { + case CopySelectionResponse.V1(response) => + Some(response.selectedCopy) + case _ => throw CopyServiceException(CopyServiceErrorCode.VersionNotFound) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.scala new file mode 100644 index 0000000000..dba016e6c0 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.scala @@ -0,0 +1,58 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.cr_mixer.thriftscala.CrMixer +import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest +import com.twitter.cr_mixer.thriftscala.CrMixerTweetResponse +import com.twitter.cr_mixer.thriftscala.FrsTweetRequest +import com.twitter.cr_mixer.thriftscala.FrsTweetResponse +import com.twitter.finagle.stats.NullStatsReceiver +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.util.Future + +/** + * Store to get content recs from content recommender. + */ +case class CrMixerTweetStore( + crMixer: CrMixer.MethodPerEndpoint +)( + implicit statsReceiver: StatsReceiver = NullStatsReceiver) { + + private val requestsCounter = statsReceiver.counter("requests") + private val successCounter = statsReceiver.counter("success") + private val failuresCounter = statsReceiver.counter("failures") + private val nonEmptyCounter = statsReceiver.counter("non_empty") + private val emptyCounter = statsReceiver.counter("empty") + private val failuresScope = statsReceiver.scope("failures") + private val latencyStat = statsReceiver.stat("latency") + + private def updateStats[T](f: => Future[Option[T]]): Future[Option[T]] = { + requestsCounter.incr() + Stat + .timeFuture(latencyStat)(f) + .onSuccess { r => + if (r.isDefined) nonEmptyCounter.incr() else emptyCounter.incr() + successCounter.incr() + } + .onFailure { e => + { + failuresCounter.incr() + failuresScope.counter(e.getClass.getName).incr() + } + } + } + + def getTweetRecommendations( + request: CrMixerTweetRequest + ): Future[Option[CrMixerTweetResponse]] = { + updateStats(crMixer.getTweetRecommendations(request).map { response => + Some(response) + }) + } + + def getFRSTweetCandidates(request: FrsTweetRequest): Future[Option[FrsTweetResponse]] = { + updateStats(crMixer.getFrsBasedTweetRecommendations(request).map { response => + Some(response) + }) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.scala new file mode 100644 index 0000000000..eeeb62d27b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.scala @@ -0,0 +1,28 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.explore_ranker.thriftscala.ExploreRanker +import com.twitter.explore_ranker.thriftscala.ExploreRankerResponse +import com.twitter.explore_ranker.thriftscala.ExploreRankerRequest +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +/** A Store for Video Tweet Recommendations from Explore + * + * @param exploreRankerService + */ +case class ExploreRankerStore(exploreRankerService: ExploreRanker.MethodPerEndpoint) + extends ReadableStore[ExploreRankerRequest, ExploreRankerResponse] { + + /** Method to get video recommendations + * + * @param request explore ranker request object + * @return + */ + override def get( + request: ExploreRankerRequest + ): Future[Option[ExploreRankerResponse]] = { + exploreRankerService.getRankedResults(request).map { response => + Some(response) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.scala new file mode 100644 index 0000000000..0ab722cd4f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.scala @@ -0,0 +1,46 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService +import com.twitter.follow_recommendations.thriftscala.Recommendation +import com.twitter.follow_recommendations.thriftscala.RecommendationRequest +import com.twitter.follow_recommendations.thriftscala.RecommendationResponse +import com.twitter.follow_recommendations.thriftscala.UserRecommendation +import com.twitter.inject.Logging +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +case class FollowRecommendationsStore( + frsClient: FollowRecommendationsThriftService.MethodPerEndpoint, + statsReceiver: StatsReceiver) + extends ReadableStore[RecommendationRequest, RecommendationResponse] + with Logging { + + private val scopedStats = statsReceiver.scope(getClass.getSimpleName) + private val requests = scopedStats.counter("requests") + private val valid = scopedStats.counter("valid") + private val invalid = scopedStats.counter("invalid") + private val numTotalResults = scopedStats.stat("total_results") + private val numValidResults = scopedStats.stat("valid_results") + + override def get(request: RecommendationRequest): Future[Option[RecommendationResponse]] = { + requests.incr() + frsClient.getRecommendations(request).map { response => + numTotalResults.add(response.recommendations.size) + val validRecs = response.recommendations.filter { + case Recommendation.User(_: UserRecommendation) => + valid.incr() + true + case _ => + invalid.incr() + false + } + + numValidResults.add(validRecs.size) + Some( + RecommendationResponse( + recommendations = validRecs + )) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.scala new file mode 100644 index 0000000000..6c355c505e --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.scala @@ -0,0 +1,190 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.finagle.stats.BroadcastStatsReceiver +import com.twitter.finagle.stats.NullStatsReceiver +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.logger.MRLogger +import com.twitter.frigate.common.store +import com.twitter.frigate.common.store.Fail +import com.twitter.frigate.common.store.IbisRequestInfo +import com.twitter.frigate.common.store.IbisResponse +import com.twitter.frigate.common.store.Sent +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.ibis2.service.thriftscala.Flags +import com.twitter.ibis2.service.thriftscala.FlowControl +import com.twitter.ibis2.service.thriftscala.Ibis2Request +import com.twitter.ibis2.service.thriftscala.Ibis2Response +import com.twitter.ibis2.service.thriftscala.Ibis2ResponseStatus +import com.twitter.ibis2.service.thriftscala.Ibis2Service +import com.twitter.ibis2.service.thriftscala.NotificationNotSentCode +import com.twitter.ibis2.service.thriftscala.TargetFanoutResult.NotSentReason +import com.twitter.util.Future + +trait Ibis2Store extends store.Ibis2Store { + def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] +} + +case class PushIbis2Store( + ibisClient: Ibis2Service.MethodPerEndpoint +)( + implicit val statsReceiver: StatsReceiver = NullStatsReceiver) + extends Ibis2Store { + private val log = MRLogger(this.getClass.getSimpleName) + private val stats = statsReceiver.scope("ibis_v2_store") + private val statsByCrt = stats.scope("byCrt") + private val requestsByCrt = statsByCrt.scope("requests") + private val failuresByCrt = statsByCrt.scope("failures") + private val successByCrt = statsByCrt.scope("success") + + private val statsByIbisModel = stats.scope("byIbisModel") + private val requestsByIbisModel = statsByIbisModel.scope("requests") + private val failuresByIbisModel = statsByIbisModel.scope("failures") + private val successByIbisModel = statsByIbisModel.scope("success") + + private[this] def ibisSend( + ibis2Request: Ibis2Request, + commonRecommendationType: CommonRecommendationType + ): Future[IbisResponse] = { + val ibisModel = ibis2Request.modelName + + val bStats = if (ibis2Request.flags.getOrElse(Flags()).darkWrite.contains(true)) { + BroadcastStatsReceiver( + Seq( + stats, + stats.scope("dark_write") + ) + ) + } else BroadcastStatsReceiver(Seq(stats)) + + bStats.counter("requests").incr() + requestsByCrt.counter(commonRecommendationType.name).incr() + requestsByIbisModel.counter(ibisModel).incr() + + retry(ibisClient, ibis2Request, 3, bStats) + .map { response => + bStats.counter(response.status.status.name).incr() + successByCrt.counter(response.status.status.name, commonRecommendationType.name).incr() + successByIbisModel.counter(response.status.status.name, ibisModel).incr() + response.status.status match { + case Ibis2ResponseStatus.SuccessWithDeliveries | + Ibis2ResponseStatus.SuccessNoDeliveries => + IbisResponse(Sent, Some(response)) + case _ => + IbisResponse(Fail, Some(response)) + } + } + .onFailure { ex => + bStats.counter("failures").incr() + val exceptionName = ex.getClass.getCanonicalName + bStats.scope("failures").counter(exceptionName).incr() + failuresByCrt.counter(exceptionName, commonRecommendationType.name).incr() + failuresByIbisModel.counter(exceptionName, ibisModel).incr() + } + } + + private def getNotifNotSentReason( + ibis2Response: Ibis2Response + ): Option[NotificationNotSentCode] = { + ibis2Response.status.fanoutResults match { + case Some(fanoutResult) => + fanoutResult.pushResult.flatMap { pushResult => + pushResult.results.headOption match { + case Some(NotSentReason(notSentInfo)) => Some(notSentInfo.notSentCode) + case _ => None + } + } + case _ => None + } + } + + def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] = { + val requestWithIID = if (ibis2Request.flowControl.exists(_.externalIid.isDefined)) { + ibis2Request + } else { + ibis2Request.copy( + flowControl = Some( + ibis2Request.flowControl + .getOrElse(FlowControl()) + .copy(externalIid = Some(candidate.impressionId)) + ) + ) + } + + val commonRecommendationType = candidate.frigateNotification.commonRecommendationType + + ibisSend(requestWithIID, commonRecommendationType) + .onSuccess { response => + response.ibis2Response.foreach { ibis2Response => + getNotifNotSentReason(ibis2Response).foreach { notifNotSentCode => + stats.scope(ibis2Response.status.status.name).counter(s"$notifNotSentCode").incr() + } + if (ibis2Response.status.status != Ibis2ResponseStatus.SuccessWithDeliveries) { + log.warning( + s"Request dropped on ibis for ${ibis2Request.recipientSelector.recipientId}: $ibis2Response") + } + } + } + .onFailure { ex => + log.warning( + s"Ibis Request failure: ${ex.getClass.getCanonicalName} \n For IbisRequest: $ibis2Request") + log.error(ex, ex.getMessage) + } + } + + // retry request when Ibis2ResponseStatus is PreFanoutError + def retry( + ibisClient: Ibis2Service.MethodPerEndpoint, + request: Ibis2Request, + retryCount: Int, + bStats: StatsReceiver + ): Future[Ibis2Response] = { + ibisClient.sendNotification(request).flatMap { response => + response.status.status match { + case Ibis2ResponseStatus.PreFanoutError if retryCount > 0 => + bStats.scope("requests").counter("retry").incr() + bStats.counter(response.status.status.name).incr() + retry(ibisClient, request, retryCount - 1, bStats) + case _ => + Future.value(response) + } + } + } + + override def send( + ibis2Request: Ibis2Request, + requestInfo: IbisRequestInfo + ): Future[IbisResponse] = { + ibisSend(ibis2Request, requestInfo.commonRecommendationType) + } +} + +case class StagingIbis2Store(remoteIbis2Store: PushIbis2Store) extends Ibis2Store { + + final def addDarkWriteFlagIbis2Request( + isTeamMember: Boolean, + ibis2Request: Ibis2Request + ): Ibis2Request = { + val flags = + ibis2Request.flags.getOrElse(Flags()) + val darkWrite: Boolean = !isTeamMember || flags.darkWrite.getOrElse(false) + ibis2Request.copy(flags = Some(flags.copy(darkWrite = Some(darkWrite)))) + } + + override def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] = { + candidate.target.isTeamMember.flatMap { isTeamMember => + val ibis2Req = addDarkWriteFlagIbis2Request(isTeamMember, ibis2Request) + remoteIbis2Store.send(ibis2Req, candidate) + } + } + + override def send( + ibis2Request: Ibis2Request, + requestInfo: IbisRequestInfo + ): Future[IbisResponse] = { + requestInfo.isTeamMember.flatMap { isTeamMember => + val ibis2Req = addDarkWriteFlagIbis2Request(isTeamMember, ibis2Request) + remoteIbis2Store.send(ibis2Req, requestInfo) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.scala new file mode 100644 index 0000000000..80fc0ea7ee --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.scala @@ -0,0 +1,16 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.interests_discovery.thriftscala.InterestsDiscoveryService +import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest +import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +case class InterestDiscoveryStore( + client: InterestsDiscoveryService.MethodPerEndpoint) + extends ReadableStore[RecommendedListsRequest, RecommendedListsResponse] { + + override def get(request: RecommendedListsRequest): Future[Option[RecommendedListsResponse]] = { + client.getListRecos(request).map(Some(_)) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.scala new file mode 100644 index 0000000000..73fc28837e --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.scala @@ -0,0 +1,156 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.candidate.TargetDecider +import com.twitter.frigate.common.history.History +import com.twitter.frigate.common.history.HistoryStoreKeyContext +import com.twitter.frigate.common.history.PushServiceHistoryStore +import com.twitter.frigate.data_pipeline.thriftscala._ +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.hermit.store.labeled_push_recs.LabeledPushRecsJoinedWithNotificationHistoryStore +import com.twitter.logging.Logger +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future +import com.twitter.util.Time + +case class LabeledPushRecsVerifyingStoreKey( + historyStoreKey: HistoryStoreKeyContext, + useHydratedDataset: Boolean, + verifyHydratedDatasetResults: Boolean) { + def userId: Long = historyStoreKey.targetUserId +} + +case class LabeledPushRecsVerifyingStoreResponse( + userHistory: UserHistoryValue, + unequalNotificationsUnhydratedToHydrated: Option[ + Map[(Time, FrigateNotification), FrigateNotification] + ], + missingFromHydrated: Option[Map[Time, FrigateNotification]]) + +case class LabeledPushRecsVerifyingStore( + labeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue], + historyStore: PushServiceHistoryStore +)( + implicit stats: StatsReceiver) + extends ReadableStore[LabeledPushRecsVerifyingStoreKey, LabeledPushRecsVerifyingStoreResponse] { + + private def getByJoiningWithRealHistory( + key: HistoryStoreKeyContext + ): Future[Option[UserHistoryValue]] = { + val historyFut = historyStore.get(key, Some(365.days)) + val toJoinWithRealHistoryFut = labeledPushRecsStore.get(UserHistoryKey.UserId(key.targetUserId)) + Future.join(historyFut, toJoinWithRealHistoryFut).map { + case (_, None) => None + case (History(realtimeHistoryMap), Some(uhValue)) => + Some( + LabeledPushRecsJoinedWithNotificationHistoryStore + .joinLabeledPushRecsSentWithNotificationHistory(uhValue, realtimeHistoryMap, stats) + ) + } + } + + private def processUserHistoryValue(uhValue: UserHistoryValue): Map[Time, FrigateNotification] = { + uhValue.events + .getOrElse(Nil) + .collect { + case Event( + EventType.LabeledPushRecSend, + Some(tsMillis), + Some(EventUnion.LabeledPushRecSendEvent(lprs: LabeledPushRecSendEvent)) + ) if lprs.pushRecSendEvent.frigateNotification.isDefined => + Time.fromMilliseconds(tsMillis) -> lprs.pushRecSendEvent.frigateNotification.get + } + .toMap + } + + override def get( + key: LabeledPushRecsVerifyingStoreKey + ): Future[Option[LabeledPushRecsVerifyingStoreResponse]] = { + val uhKey = UserHistoryKey.UserId(key.userId) + if (!key.useHydratedDataset) { + getByJoiningWithRealHistory(key.historyStoreKey).map { uhValueOpt => + uhValueOpt.map { uhValue => LabeledPushRecsVerifyingStoreResponse(uhValue, None, None) } + } + } else { + labeledPushRecsStore.get(uhKey).flatMap { hydratedValueOpt: Option[UserHistoryValue] => + if (!key.verifyHydratedDatasetResults) { + Future.value(hydratedValueOpt.map { uhValue => + LabeledPushRecsVerifyingStoreResponse(uhValue, None, None) + }) + } else { + getByJoiningWithRealHistory(key.historyStoreKey).map { + joinedWithRealHistoryOpt: Option[UserHistoryValue] => + val joinedWithRealHistoryMap = + joinedWithRealHistoryOpt.map(processUserHistoryValue).getOrElse(Map.empty) + val hydratedMap = hydratedValueOpt.map(processUserHistoryValue).getOrElse(Map.empty) + val unequal = joinedWithRealHistoryMap.flatMap { + case (time, frigateNotif) => + hydratedMap.get(time).collect { + case n if n != frigateNotif => ((time, frigateNotif), n) + } + } + val missing = joinedWithRealHistoryMap.filter { + case (time, frigateNotif) => !hydratedMap.contains(time) + } + hydratedValueOpt.map { hydratedValue => + LabeledPushRecsVerifyingStoreResponse(hydratedValue, Some(unequal), Some(missing)) + } + } + } + } + } + } +} + +case class LabeledPushRecsStoreKey(target: TargetDecider, historyStoreKey: HistoryStoreKeyContext) { + def userId: Long = historyStoreKey.targetUserId +} + +case class LabeledPushRecsDecideredStore( + verifyingStore: ReadableStore[ + LabeledPushRecsVerifyingStoreKey, + LabeledPushRecsVerifyingStoreResponse + ], + useHydratedLabeledSendsDatasetDeciderKey: String, + verifyHydratedLabeledSendsForHistoryDeciderKey: String +)( + implicit globalStats: StatsReceiver) + extends ReadableStore[LabeledPushRecsStoreKey, UserHistoryValue] { + private val log = Logger() + private val stats = globalStats.scope("LabeledPushRecsDecideredStore") + private val numComparisons = stats.counter("num_comparisons") + private val numMissingStat = stats.stat("num_missing") + private val numUnequalStat = stats.stat("num_unequal") + + override def get(key: LabeledPushRecsStoreKey): Future[Option[UserHistoryValue]] = { + val useHydrated = key.target.isDeciderEnabled( + useHydratedLabeledSendsDatasetDeciderKey, + stats, + useRandomRecipient = true + ) + + val verifyHydrated = if (useHydrated) { + key.target.isDeciderEnabled( + verifyHydratedLabeledSendsForHistoryDeciderKey, + stats, + useRandomRecipient = true + ) + } else false + + val newKey = LabeledPushRecsVerifyingStoreKey(key.historyStoreKey, useHydrated, verifyHydrated) + verifyingStore.get(newKey).map { + case None => None + case Some(LabeledPushRecsVerifyingStoreResponse(uhValue, unequalOpt, missingOpt)) => + (unequalOpt, missingOpt) match { + case (Some(unequal), Some(missing)) => + numComparisons.incr() + numMissingStat.add(missing.size) + numUnequalStat.add(unequal.size) + case _ => //no-op + } + Some(uhValue) + } + } + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.scala new file mode 100644 index 0000000000..b11cdc0dd9 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.scala @@ -0,0 +1,26 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.livevideo.common.ids.EventId +import com.twitter.livevideo.timeline.client.v2.LiveVideoTimelineClient +import com.twitter.livevideo.timeline.domain.v2.Event +import com.twitter.livevideo.timeline.domain.v2.LookupContext +import com.twitter.stitch.storehaus.ReadableStoreOfStitch +import com.twitter.stitch.NotFound +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore + +case class EventRequest(eventId: Long, lookupContext: LookupContext = LookupContext.default) + +object LexServiceStore { + def apply( + liveVideoTimelineClient: LiveVideoTimelineClient + ): ReadableStore[EventRequest, Event] = { + ReadableStoreOfStitch { eventRequest => + liveVideoTimelineClient.getEvent( + EventId(eventRequest.eventId), + eventRequest.lookupContext) rescue { + case NotFound => Stitch.NotFound + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.scala new file mode 100644 index 0000000000..9e9bc37b73 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.scala @@ -0,0 +1,45 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.hermit.store.common.ReadableWritableStore +import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey +import com.twitter.stitch.Stitch +import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryCompactScalaInjection +import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection +import com.twitter.storage.client.manhattan.bijections.Bijections.StringInjection +import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint +import com.twitter.storage.client.manhattan.kv.impl.Component +import com.twitter.storage.client.manhattan.kv.impl.DescriptorP1L1 +import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor +import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor +import com.twitter.util.Future + +case class NTabHistoryStore(mhEndpoint: ManhattanKVEndpoint, dataset: String) + extends ReadableWritableStore[(Long, String), GenericNotificationOverrideKey] { + + private val keyDesc: DescriptorP1L1.EmptyKey[Long, String] = + KeyDescriptor(Component(LongInjection), Component(StringInjection)) + + private val genericNotifKeyValDesc: ValueDescriptor.EmptyValue[GenericNotificationOverrideKey] = + ValueDescriptor[GenericNotificationOverrideKey]( + BinaryCompactScalaInjection(GenericNotificationOverrideKey) + ) + + override def get(key: (Long, String)): Future[Option[GenericNotificationOverrideKey]] = { + val (userId, impressionId) = key + val mhKey = keyDesc.withDataset(dataset).withPkey(userId).withLkey(impressionId) + + Stitch + .run(mhEndpoint.get(mhKey, genericNotifKeyValDesc)) + .map { optionMhValue => + optionMhValue.map(_.contents) + } + } + + override def put(keyValue: ((Long, String), GenericNotificationOverrideKey)): Future[Unit] = { + val ((userId, impressionId), genericNotifOverrideKey) = keyValue + val mhKey = keyDesc.withDataset(dataset).withPkey(userId).withLkey(impressionId) + val mhVal = genericNotifKeyValDesc.withValue(genericNotifOverrideKey) + Stitch.run(mhEndpoint.insert(mhKey, mhVal)) + } + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.scala new file mode 100644 index 0000000000..33b119c798 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.scala @@ -0,0 +1,73 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment +import com.twitter.stitch.Stitch +import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryScalaInjection +import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection +import com.twitter.storage.client.manhattan.bijections.Bijections.StringInjection +import com.twitter.storage.client.manhattan.kv.impl.Component +import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor +import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor +import com.twitter.storage.client.manhattan.kv.ManhattanKVClient +import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams +import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder +import com.twitter.storage.client.manhattan.kv.NoMtlsParams +import com.twitter.storehaus.ReadableStore +import com.twitter.storehaus_internal.manhattan.Omega +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.util.Time + +case class OCFHistoryStoreKey(userId: Long, fatigueDuration: Duration, fatigueGroup: String) + +class OCFPromptHistoryStore( + manhattanAppId: String, + dataset: String, + mtlsParams: ManhattanKVClientMtlsParams = NoMtlsParams +)( + implicit stats: StatsReceiver) + extends ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] { + + import ManhattanInjections._ + + private val client = ManhattanKVClient( + appId = manhattanAppId, + dest = Omega.wilyName, + mtlsParams = mtlsParams, + label = "ocf_history_store" + ) + private val endpoint = ManhattanKVEndpointBuilder(client, defaultMaxTimeout = 5.seconds) + .statsReceiver(stats.scope("ocf_history_store")) + .build() + + private val limitResultsTo = 1 + + private val datasetKey = keyDesc.withDataset(dataset) + + override def get(storeKey: OCFHistoryStoreKey): Future[Option[FatigueFlowEnrollment]] = { + val userId = storeKey.userId + val fatigueGroup = storeKey.fatigueGroup + val fatigueLength = storeKey.fatigueDuration.inMilliseconds + val currentTime = Time.now.inMilliseconds + val fullKey = datasetKey + .withPkey(userId) + .from(fatigueGroup) + .to(fatigueGroup, fatigueLength - currentTime) + + Stitch + .run(endpoint.slice(fullKey, valDesc, limit = Some(limitResultsTo))) + .map { results => + if (results.nonEmpty) { + val (_, mhValue) = results.head + Some(mhValue.contents) + } else None + } + } +} + +object ManhattanInjections { + val keyDesc = KeyDescriptor(Component(LongInjection), Component(StringInjection, LongInjection)) + val valDesc = ValueDescriptor(BinaryScalaInjection(FatigueFlowEnrollment)) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.scala new file mode 100644 index 0000000000..d7ecfa7e4b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.scala @@ -0,0 +1,81 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.conversions.DurationOps._ +import com.twitter.frigate.common.history.History +import com.twitter.frigate.common.store.RealTimeClientEventStore +import com.twitter.frigate.data_pipeline.common.HistoryJoin +import com.twitter.frigate.data_pipeline.thriftscala.Event +import com.twitter.frigate.data_pipeline.thriftscala.EventUnion +import com.twitter.frigate.data_pipeline.thriftscala.PushRecSendEvent +import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.util.Time + +case class OnlineUserHistoryKey( + userId: Long, + offlineUserHistory: Option[UserHistoryValue], + history: Option[History]) + +case class OnlineUserHistoryStore( + realTimeClientEventStore: RealTimeClientEventStore, + duration: Duration = 3.days) + extends ReadableStore[OnlineUserHistoryKey, UserHistoryValue] { + + override def get(key: OnlineUserHistoryKey): Future[Option[UserHistoryValue]] = { + val now = Time.now + + val pushRecSends = key.history + .getOrElse(History(Nil.toMap)) + .sortedPushDmHistory + .filter(_._1 > now - (duration + 1.day)) + .map { + case (time, frigateNotification) => + val pushRecSendEvent = PushRecSendEvent( + frigateNotification = Some(frigateNotification), + impressionId = frigateNotification.impressionId + ) + pushRecSendEvent -> time + } + + realTimeClientEventStore + .get(key.userId, now - duration, now) + .map { attributedEventHistory => + val attributedClientEvents = attributedEventHistory.sortedHistory.flatMap { + case (time, event) => + event.eventUnion match { + case Some(eventUnion: EventUnion.AttributedPushRecClientEvent) => + Some((eventUnion.attributedPushRecClientEvent, event.eventType, time)) + case _ => None + } + } + + val realtimeLabeledSends: Seq[Event] = HistoryJoin.getLabeledPushRecSends( + pushRecSends, + attributedClientEvents, + Seq(), + Seq(), + Seq(), + now + ) + + key.offlineUserHistory.map { offlineUserHistory => + val combinedEvents = offlineUserHistory.events.map { offlineEvents => + (offlineEvents ++ realtimeLabeledSends) + .map { event => + event.timestampMillis -> event + } + .toMap + .values + .toSeq + .sortBy { event => + -1 * event.timestampMillis.getOrElse(0L) + } + } + + offlineUserHistory.copy(events = combinedEvents) + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.scala new file mode 100644 index 0000000000..85d3f5afa1 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.scala @@ -0,0 +1,13 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.frigate.common.store.strato.StratoFetchableStore +import com.twitter.storehaus.ReadableStore +import com.twitter.strato.client.Client +import com.twitter.strato.generated.client.rux.open_app.UsersInOpenAppDdgOnUserClientColumn + +object OpenAppUserStore { + def apply(stratoClient: Client): ReadableStore[Long, Boolean] = { + val fetcher = new UsersInOpenAppDdgOnUserClientColumn(stratoClient).fetcher + StratoFetchableStore.withUnitView(fetcher).mapValues(_ => true) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.scala new file mode 100644 index 0000000000..4af473656c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.scala @@ -0,0 +1,21 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.frigate.pushservice.params.PushQPSLimitConstants.SocialGraphServiceBatchSize +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +case class SocialGraphServiceProcessStore(edgeStore: ReadableStore[RelationEdge, Boolean]) + extends ReadableStore[RelationEdge, Boolean] { + override def multiGet[T <: RelationEdge]( + relationEdges: Set[T] + ): Map[T, Future[Option[Boolean]]] = { + val splitSet = relationEdges.grouped(SocialGraphServiceBatchSize).toSet + splitSet + .map { relationship => + edgeStore.multiGet(relationship) + }.foldLeft(Map.empty[T, Future[Option[Boolean]]]) { (map1, map2) => + map1 ++ map2 + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.scala new file mode 100644 index 0000000000..b2de4cf26c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.scala @@ -0,0 +1,61 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.gizmoduck.thriftscala.UserType +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.strato.client.Client +import com.twitter.strato.client.UserId +import com.twitter.strato.config.FlockCursors.BySource.Begin +import com.twitter.strato.config.FlockCursors.Continue +import com.twitter.strato.config.FlockCursors.End +import com.twitter.strato.config.FlockPage +import com.twitter.strato.generated.client.socialgraph.service.soft_users.softUserFollows.EdgeBySourceClientColumn +import com.twitter.util.Future + +object SoftUserFollowingStore { + type ViewerFollowingCursor = EdgeBySourceClientColumn.Cursor + val MaxPagesToFetch = 2 + val PageLimit = 50 +} + +class SoftUserFollowingStore(stratoClient: Client) extends ReadableStore[User, Seq[Long]] { + import SoftUserFollowingStore._ + private val softUserFollowingEdgesPaginator = new EdgeBySourceClientColumn(stratoClient).paginator + + private def accumulateIds(cursor: ViewerFollowingCursor, pagesToFetch: Int): Stitch[Seq[Long]] = + softUserFollowingEdgesPaginator.paginate(cursor).flatMap { + case FlockPage(data, next, _) => + next match { + case cont: Continue if pagesToFetch > 1 => + Stitch + .join( + Stitch.value(data.map(_.to).map(_.value)), + accumulateIds(cont, pagesToFetch - 1)) + .map { + case (a, b) => a ++ b + } + + case _: End | _: Continue => + // end pagination if last page has been fetched or [[MaxPagesToFetch]] have been fetched + Stitch.value(data.map(_.to).map(_.value)) + } + } + + private def softFollowingFromStrato( + sourceId: Long, + pageLimit: Int, + pagesToFetch: Int + ): Stitch[Seq[Long]] = { + val begin = Begin[UserId, UserId](UserId(sourceId), pageLimit) + accumulateIds(begin, pagesToFetch) + } + + override def get(user: User): Future[Option[Seq[Long]]] = { + user.userType match { + case UserType.Soft => + Stitch.run(softFollowingFromStrato(user.id, PageLimit, MaxPagesToFetch)).map(Option(_)) + case _ => Future.None + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.scala new file mode 100644 index 0000000000..6acf1d136c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.scala @@ -0,0 +1,19 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.frigate.common.store.strato.StratoFetchableStore +import com.twitter.storehaus.ReadableStore +import com.twitter.strato.client.{Client => StratoClient} +import com.twitter.util.Future + +/** + * Store to get inbound Tweet impressions count for a specific Tweet id. + */ +class TweetImpressionsStore(stratoClient: StratoClient) extends ReadableStore[Long, String] { + + private val column = "rux/impression.Tweet" + private val store = StratoFetchableStore.withUnitView[Long, String](stratoClient, column) + + def getCounts(tweetId: Long): Future[Option[Long]] = { + store.get(tweetId).map(_.map(_.toLong)) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.scala new file mode 100644 index 0000000000..618d8da329 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.scala @@ -0,0 +1,211 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.context.TwitterContext +import com.twitter.context.thriftscala.Viewer +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.TwitterContextPermit +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.kujaku.domain.thriftscala.CacheUsageType +import com.twitter.kujaku.domain.thriftscala.MachineTranslation +import com.twitter.kujaku.domain.thriftscala.MachineTranslationResponse +import com.twitter.kujaku.domain.thriftscala.TranslationSource +import com.twitter.storehaus.ReadableStore +import com.twitter.strato.generated.client.translation.service.IsTweetTranslatableClientColumn +import com.twitter.strato.generated.client.translation.service.platform.MachineTranslateTweetClientColumn +import com.twitter.tweetypie.thriftscala.Tweet +import com.twitter.util.Future +import com.twitter.util.logging.Logging + +object TweetTranslationStore { + case class Key( + target: Target, + tweetId: Long, + tweet: Option[Tweet], + crt: CommonRecommendationType) + + case class Value( + translatedTweetText: String, + localizedSourceLanguage: String) + + val allowedCRTs = Set[CommonRecommendationType]( + CommonRecommendationType.TwistlyTweet + ) +} + +case class TweetTranslationStore( + translateTweetStore: ReadableStore[ + MachineTranslateTweetClientColumn.Key, + MachineTranslationResponse + ], + isTweetTranslatableStore: ReadableStore[IsTweetTranslatableClientColumn.Key, Boolean], + statsReceiver: StatsReceiver) + extends ReadableStore[TweetTranslationStore.Key, TweetTranslationStore.Value] + with Logging { + + private val stats = statsReceiver.scope("tweetTranslationStore") + private val isTranslatableCounter = stats.counter("tweetIsTranslatable") + private val notTranslatableCounter = stats.counter("tweetIsNotTranslatable") + private val protectedUserCounter = stats.counter("protectedUser") + private val notProtectedUserCounter = stats.counter("notProtectedUser") + private val validLanguageCounter = stats.counter("validTweetLanguage") + private val invalidLanguageCounter = stats.counter("invalidTweetLanguage") + private val validCrtCounter = stats.counter("validCrt") + private val invalidCrtCounter = stats.counter("invalidCrt") + private val paramEnabledCounter = stats.counter("paramEnabled") + private val paramDisabledCounter = stats.counter("paramDisabled") + + private val twitterContext = TwitterContext(TwitterContextPermit) + + override def get(k: TweetTranslationStore.Key): Future[Option[TweetTranslationStore.Value]] = { + k.target.inferredUserDeviceLanguage.flatMap { + case Some(deviceLanguage) => + setTwitterContext(k.target, deviceLanguage) { + translateTweet( + target = k.target, + tweetId = k.tweetId, + tweet = k.tweet, + crt = k.crt, + deviceLanguage = deviceLanguage).map { responseOpt => + responseOpt.flatMap { response => + response.translatorLocalizedSourceLanguage + .map { localizedSourceLanguage => + TweetTranslationStore.Value( + translatedTweetText = response.translation, + localizedSourceLanguage = localizedSourceLanguage + ) + }.filter { _ => + response.translationSource == TranslationSource.Google + } + } + } + } + case None => Future.None + } + + } + + // Don't sent protected tweets to external API for translation + private def checkProtectedUser(target: Target): Future[Boolean] = { + target.targetUser.map(_.flatMap(_.safety).forall(_.isProtected)).onSuccess { + case true => protectedUserCounter.incr() + case false => notProtectedUserCounter.incr() + } + } + + private def isTweetTranslatable( + target: Target, + tweetId: Long, + tweet: Option[Tweet], + crt: CommonRecommendationType, + deviceLanguage: String + ): Future[Boolean] = { + val tweetLangOpt = tweet.flatMap(_.language) + val isValidLanguage = tweetLangOpt.exists { tweetLang => + tweetLang.confidence > 0.5 && + tweetLang.language != deviceLanguage + } + + if (isValidLanguage) { + validLanguageCounter.incr() + } else { + invalidLanguageCounter.incr() + } + + val isValidCrt = TweetTranslationStore.allowedCRTs.contains(crt) + if (isValidCrt) { + validCrtCounter.incr() + } else { + invalidCrtCounter.incr() + } + + if (isValidCrt && isValidLanguage && target.params(PushParams.EnableIsTweetTranslatableCheck)) { + checkProtectedUser(target).flatMap { + case false => + val isTweetTranslatableKey = IsTweetTranslatableClientColumn.Key( + tweetId = tweetId, + destinationLanguage = Some(deviceLanguage), + translationSource = Some(TranslationSource.Google.name), + excludePreferredLanguages = Some(true) + ) + isTweetTranslatableStore + .get(isTweetTranslatableKey).map { resultOpt => + resultOpt.getOrElse(false) + }.onSuccess { + case true => isTranslatableCounter.incr() + case false => notTranslatableCounter.incr() + } + case true => + Future.False + } + } else { + Future.False + } + } + + private def translateTweet( + tweetId: Long, + deviceLanguage: String + ): Future[Option[MachineTranslation]] = { + val translateKey = MachineTranslateTweetClientColumn.Key( + tweetId = tweetId, + destinationLanguage = deviceLanguage, + translationSource = TranslationSource.Google, + translatableEntityTypes = Seq(), + onlyCached = false, + cacheUsageType = CacheUsageType.Default + ) + translateTweetStore.get(translateKey).map { + _.collect { + case MachineTranslationResponse.Result(result) => result + } + } + } + + private def translateTweet( + target: Target, + tweetId: Long, + tweet: Option[Tweet], + crt: CommonRecommendationType, + deviceLanguage: String + ): Future[Option[MachineTranslation]] = { + isTweetTranslatable(target, tweetId, tweet, crt, deviceLanguage).flatMap { + case true => + val isEnabledByParam = target.params(PushFeatureSwitchParams.EnableTweetTranslation) + if (isEnabledByParam) { + paramEnabledCounter.incr() + translateTweet(tweetId, deviceLanguage) + } else { + paramDisabledCounter.incr() + Future.None + } + case false => + Future.None + } + } + + private def setTwitterContext[Rep]( + target: Target, + deviceLanguage: String + )( + f: => Future[Rep] + ): Future[Rep] = { + twitterContext() match { + case Some(viewer) if viewer.userId.nonEmpty && viewer.authenticatedUserId.nonEmpty => + // If the context is already setup with a user ID just use it + f + case _ => + // If not, create a new context containing the viewer user id + twitterContext.let( + Viewer( + userId = Some(target.targetId), + requestLanguageCode = Some(deviceLanguage), + authenticatedUserId = Some(target.targetId) + )) { + f + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.scala new file mode 100644 index 0000000000..bd96bf690a --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.scala @@ -0,0 +1,79 @@ +package com.twitter.frigate.pushservice.store + +import com.twitter.escherbird.util.uttclient.CachedUttClientV2 +import com.twitter.escherbird.util.uttclient.InvalidUttEntityException +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.logging.Logger +import com.twitter.stitch.Stitch +import com.twitter.topiclisting.TopicListingViewerContext +import com.twitter.topiclisting.utt.LocalizedEntity +import com.twitter.topiclisting.utt.LocalizedEntityFactory +import com.twitter.util.Future + +/** + * + * @param viewerContext: [[TopicListingViewerContext]] for filtering topic + * @param semanticCoreEntityIds: list of semantic core entities to hydrate + */ +case class UttEntityHydrationQuery( + viewerContext: TopicListingViewerContext, + semanticCoreEntityIds: Seq[Long]) + +/** + * + * @param cachedUttClientV2 + * @param statsReceiver + */ +class UttEntityHydrationStore( + cachedUttClientV2: CachedUttClientV2, + statsReceiver: StatsReceiver, + log: Logger) { + + private val stats = statsReceiver.scope(this.getClass.getSimpleName) + private val uttEntityNotFound = stats.counter("invalid_utt_entity") + private val deviceLanguageMismatch = stats.counter("language_mismatch") + + /** + * SemanticCore recommends setting language and country code to None to fetch all localized topic + * names and apply filtering for locales on our end + * + * We use [[LocalizedEntityFactory]] from [[Topiclisting]] library to filter out topic name based + * on user locale + * + * Some(LocalizedEntity) - LocalizedUttEntity found + * None - LocalizedUttEntity not found + */ + def getLocalizedTopicEntities( + query: UttEntityHydrationQuery + ): Future[Seq[Option[LocalizedEntity]]] = Stitch.run { + Stitch.collect { + query.semanticCoreEntityIds.map { semanticCoreEntityId => + val uttEntity = cachedUttClientV2.cachedGetUttEntity( + language = None, + country = None, + version = None, + entityId = semanticCoreEntityId) + + uttEntity + .map { uttEntityMetadata => + val localizedEntity = LocalizedEntityFactory.getLocalizedEntity( + uttEntityMetadata, + query.viewerContext, + enableInternationalTopics = true, + enableTopicDescription = true) + // update counter + localizedEntity.foreach { entity => + if (!entity.nameMatchesDeviceLanguage) deviceLanguageMismatch.incr() + } + + localizedEntity + }.handle { + case e: InvalidUttEntityException => + log.error(e.getMessage) + uttEntityNotFound.incr() + None + } + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.scala new file mode 100644 index 0000000000..1eb8cbc046 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.scala @@ -0,0 +1,160 @@ +package com.twitter.frigate.pushservice.take + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.Stats.track +import com.twitter.frigate.common.logger.MRLogger +import com.twitter.frigate.common.store.Fail +import com.twitter.frigate.common.store.IbisResponse +import com.twitter.frigate.common.store.InvalidConfiguration +import com.twitter.frigate.common.store.NoRequest +import com.twitter.frigate.common.store.Sent +import com.twitter.frigate.common.util.CasLock +import com.twitter.frigate.common.util.PushServiceUtil.InvalidConfigResponse +import com.twitter.frigate.common.util.PushServiceUtil.NtabWriteOnlyResponse +import com.twitter.frigate.common.util.PushServiceUtil.SendFailedResponse +import com.twitter.frigate.common.util.PushServiceUtil.SentResponse +import com.twitter.frigate.pushservice.predicate.CasLockPredicate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.take.history._ +import com.twitter.frigate.pushservice.util.CopyUtil +import com.twitter.frigate.pushservice.thriftscala.PushResponse +import com.twitter.frigate.pushservice.thriftscala.PushStatus +import com.twitter.frigate.pushservice.util.OverrideNotificationUtil +import com.twitter.frigate.thriftscala.ChannelName +import com.twitter.util.Future + +class CandidateNotifier( + notificationSender: NotificationSender, + casLock: CasLock, + historyWriter: HistoryWriter, + eventBusWriter: EventBusWriter, + ntabOnlyChannelSelector: NtabOnlyChannelSelector +)( + implicit statsReceiver: StatsReceiver) { + + private lazy val casLockPredicate = + CasLockPredicate(casLock, expiryDuration = 10.minutes)(statsReceiver) + private val candidateNotifierStats = statsReceiver.scope(this.getClass.getSimpleName) + private val historyWriteCounter = + candidateNotifierStats.counter("simply_notifier_history_write_num") + private val loggedOutHistoryWriteCounter = + candidateNotifierStats.counter("logged_out_simply_notifier_history_write_num") + private val notificationSenderLatency = + candidateNotifierStats.scope("notification_sender_send") + private val log = MRLogger("CandidateNotifier") + + private def mapIbisResponse(ibisResponse: IbisResponse): PushResponse = { + ibisResponse match { + case IbisResponse(Sent, _) => SentResponse + case IbisResponse(Fail, _) => SendFailedResponse + case IbisResponse(InvalidConfiguration, _) => InvalidConfigResponse + case IbisResponse(NoRequest, _) => NtabWriteOnlyResponse + } + } + + /** + * - write to history store + * - send the notification + * - scribe the notification + * + * final modifier is to signal that this function cannot be overriden. There's some critical logic + * in this function, and it's helpful to know that no sub-class overrides it. + */ + final def notify( + candidate: PushCandidate, + ): Future[PushResponse] = { + if (candidate.target.isDarkWrite) { + notificationSender.sendIbisDarkWrite(candidate).map(mapIbisResponse) + } else { + casLockPredicate(Seq(candidate)).flatMap { casLockResults => + if (casLockResults.head || candidate.target.pushContext + .exists(_.skipFilters.contains(true))) { + Future + .join( + candidate.target.isSilentPush, + OverrideNotificationUtil + .getOverrideInfo(candidate, candidateNotifierStats), + CopyUtil.getCopyFeatures(candidate, candidateNotifierStats) + ).flatMap { + case (isSilentPush, overrideInfoOpt, copyFeaturesMap) => + val channels = ntabOnlyChannelSelector.selectChannel(candidate) + channels.flatMap { channels => + candidate + .frigateNotificationForPersistence( + channels, + isSilentPush, + overrideInfoOpt, + copyFeaturesMap.keySet).flatMap { frigateNotificationForPersistence => + val result = if (candidate.target.isDarkWrite) { + candidateNotifierStats.counter("dark_write").incr() + Future.Unit + } else { + historyWriteCounter.incr() + historyWriter + .writeSendToHistory(candidate, frigateNotificationForPersistence) + } + result.flatMap { _ => + track(notificationSenderLatency)( + notificationSender + .notify(channels, candidate) + .map { ibisResponse => + eventBusWriter + .writeToEventBus(candidate, frigateNotificationForPersistence) + mapIbisResponse(ibisResponse) + }) + } + } + } + } + } else { + candidateNotifierStats.counter("filtered_by_cas_lock").incr() + Future.value(PushResponse(PushStatus.Filtered, Some(casLockPredicate.name))) + } + } + } + } + + final def loggedOutNotify( + candidate: PushCandidate, + ): Future[PushResponse] = { + if (candidate.target.isDarkWrite) { + notificationSender.sendIbisDarkWrite(candidate).map(mapIbisResponse) + } else { + casLockPredicate(Seq(candidate)).flatMap { casLockResults => + if (casLockResults.head || candidate.target.pushContext + .exists(_.skipFilters.contains(true))) { + val response = candidate.target.isSilentPush.flatMap { isSilentPush => + candidate + .frigateNotificationForPersistence( + Seq(ChannelName.PushNtab), + isSilentPush, + None, + Set.empty).flatMap { frigateNotificationForPersistence => + val result = if (candidate.target.isDarkWrite) { + candidateNotifierStats.counter("logged_out_dark_write").incr() + Future.Unit + } else { + loggedOutHistoryWriteCounter.incr() + historyWriter.writeSendToHistory(candidate, frigateNotificationForPersistence) + } + + result.flatMap { _ => + track(notificationSenderLatency)( + notificationSender + .loggedOutNotify(candidate) + .map { ibisResponse => + mapIbisResponse(ibisResponse) + }) + } + } + } + response + } else { + candidateNotifierStats.counter("filtered_by_cas_lock").incr() + Future.value(PushResponse(PushStatus.Filtered, Some(casLockPredicate.name))) + } + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.scala new file mode 100644 index 0000000000..07574f46f1 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.scala @@ -0,0 +1,118 @@ +package com.twitter.frigate.pushservice.take + +import com.twitter.finagle.stats.BroadcastStatsReceiver +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateResult +import com.twitter.frigate.common.base.Invalid +import com.twitter.frigate.common.base.OK +import com.twitter.frigate.common.base.Response +import com.twitter.frigate.common.base.Result +import com.twitter.frigate.common.base.Stats.track +import com.twitter.frigate.common.config.CommonConstants +import com.twitter.frigate.common.logger.MRLogger +import com.twitter.frigate.common.util.PushServiceUtil.FilteredLoggedOutResponseFut +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.refresh_handler.RFPHStatsRecorder +import com.twitter.frigate.pushservice.thriftscala.LoggedOutResponse +import com.twitter.frigate.pushservice.thriftscala.PushStatus +import com.twitter.util.Future +import com.twitter.util.JavaTimer +import com.twitter.util.Timer + +class LoggedOutRefreshForPushNotifier( + rfphStatsRecorder: RFPHStatsRecorder, + loCandidateNotifier: CandidateNotifier +)( + globalStats: StatsReceiver) { + private implicit val statsReceiver: StatsReceiver = + globalStats.scope("LoggedOutRefreshForPushHandler") + private val loPushStats: StatsReceiver = statsReceiver.scope("logged_out_push") + private val loSendLatency: StatsReceiver = statsReceiver.scope("logged_out_send") + private val processedCandidatesCounter: Counter = + statsReceiver.counter("processed_candidates_count") + private val validCandidatesCounter: Counter = statsReceiver.counter("valid_candidates_count") + private val okayCandidateCounter: Counter = statsReceiver.counter("ok_candidate_count") + private val nonOkayCandidateCounter: Counter = statsReceiver.counter("non_ok_candidate_count") + private val successNotifyCounter: Counter = statsReceiver.counter("success_notify_count") + private val notifyCandidate: Counter = statsReceiver.counter("notify_candidate") + private val noneCandidateResultCounter: Counter = statsReceiver.counter("none_candidate_count") + private val nonOkayPredsResult: Counter = statsReceiver.counter("non_okay_preds_result") + private val invalidResultCounter: Counter = statsReceiver.counter("invalid_result_count") + private val filteredLoggedOutResponse: Counter = statsReceiver.counter("filtered_response_count") + + implicit private val timer: Timer = new JavaTimer(true) + val log = MRLogger("LoggedOutRefreshForNotifier") + + private def notify( + candidatesResult: CandidateResult[PushCandidate, Result] + ): Future[LoggedOutResponse] = { + val candidate = candidatesResult.candidate + if (candidate != null) + notifyCandidate.incr() + val predsResult = candidatesResult.result + if (predsResult != OK) { + nonOkayPredsResult.incr() + val invalidResult = predsResult + invalidResult match { + case Invalid(Some(reason)) => + invalidResultCounter.incr() + Future.value(LoggedOutResponse(PushStatus.Filtered, Some(reason))) + case _ => + filteredLoggedOutResponse.incr() + Future.value(LoggedOutResponse(PushStatus.Filtered, None)) + } + } else { + track(loSendLatency)(loCandidateNotifier.loggedOutNotify(candidate).map { res => + LoggedOutResponse(res.status) + }) + } + } + + def checkResponseAndNotify( + response: Response[PushCandidate, Result] + ): Future[LoggedOutResponse] = { + val receivers = Seq(statsReceiver) + val loggedOutResponse = response match { + case Response(OK, processedCandidates) => + processedCandidatesCounter.incr(processedCandidates.size) + val validCandidates = processedCandidates.filter(_.result == OK) + validCandidatesCounter.incr(validCandidates.size) + + validCandidates.headOption match { + case Some(candidatesResult) => + candidatesResult.result match { + case OK => + okayCandidateCounter.incr() + notify(candidatesResult) + .onSuccess { nr => + successNotifyCounter.incr() + loPushStats.scope("lo_result").counter(nr.status.name).incr() + } + case _ => + nonOkayCandidateCounter.incr() + FilteredLoggedOutResponseFut + } + case _ => + noneCandidateResultCounter.incr() + FilteredLoggedOutResponseFut + } + + case Response(Invalid(reason), _) => + FilteredLoggedOutResponseFut.map(_.copy(filteredBy = reason)) + + case _ => + FilteredLoggedOutResponseFut + } + val bstats = BroadcastStatsReceiver(receivers) + Stat + .timeFuture(bstats.stat("logged_out_latency"))( + loggedOutResponse.raiseWithin(CommonConstants.maxPushRequestDuration) + ) + .onFailure { exception => + rfphStatsRecorder.loggedOutRequestExceptionStats(exception, bstats) + } + loggedOutResponse + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.scala new file mode 100644 index 0000000000..70a695fb3e --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.scala @@ -0,0 +1,95 @@ +package com.twitter.frigate.pushservice.take + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.Stats.track +import com.twitter.frigate.common.store.IbisResponse +import com.twitter.frigate.common.store.Sent +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.take.sender.Ibis2Sender +import com.twitter.frigate.pushservice.take.sender.NtabSender +import com.twitter.frigate.scribe.thriftscala.NotificationScribe +import com.twitter.util.Future +import com.twitter.frigate.thriftscala.ChannelName + +/** + * NotificationSender wraps up all the notification infra send logic, and serves as an abstract layer + * between CandidateNotifier and the respective senders including ntab, ibis, which is being + * gated with both a decider/feature switch + */ +class NotificationSender( + ibis2Sender: Ibis2Sender, + ntabSender: NtabSender, + statsReceiver: StatsReceiver, + notificationScribe: NotificationScribe => Unit) { + + private val notificationNotifierStats = statsReceiver.scope(this.getClass.getSimpleName) + private val ibis2SendLatency = notificationNotifierStats.scope("ibis2_send") + private val loggedOutIbis2SendLatency = notificationNotifierStats.scope("logged_out_ibis2_send") + private val ntabSendLatency = notificationNotifierStats.scope("ntab_send") + + private val ntabWriteThenSkipPushCounter = + notificationNotifierStats.counter("ntab_write_then_skip_push") + private val ntabWriteThenIbisSendCounter = + notificationNotifierStats.counter("ntab_write_then_ibis_send") + notificationNotifierStats.counter("ins_dark_traffic_send") + + private val ntabOnlyChannelSenderV3Counter = + notificationNotifierStats.counter("ntab_only_channel_send_v3") + + def sendIbisDarkWrite(candidate: PushCandidate): Future[IbisResponse] = { + ibis2Sender.sendAsDarkWrite(candidate) + } + + private def isNtabOnlySend( + channels: Seq[ChannelName] + ): Future[Boolean] = { + val isNtabOnlyChannel = channels.contains(ChannelName.NtabOnly) + if (isNtabOnlyChannel) ntabOnlyChannelSenderV3Counter.incr() + + Future.value(isNtabOnlyChannel) + } + + private def isPushOnly(channels: Seq[ChannelName], candidate: PushCandidate): Future[Boolean] = { + Future.value(channels.contains(ChannelName.PushOnly)) + } + + def notify( + channels: Seq[ChannelName], + candidate: PushCandidate + ): Future[IbisResponse] = { + Future + .join(isPushOnly(channels, candidate), isNtabOnlySend(channels)).map { + case (isPushOnly, isNtabOnly) => + if (isPushOnly) { + track(ibis2SendLatency)(ibis2Sender.send(channels, candidate, notificationScribe, None)) + } else { + track(ntabSendLatency)( + ntabSender + .send(candidate, isNtabOnly)) + .flatMap { ntabResponse => + if (isNtabOnly) { + ntabWriteThenSkipPushCounter.incr() + candidate + .scribeData(channels = channels).foreach(notificationScribe).map(_ => + IbisResponse(Sent)) + } else { + ntabWriteThenIbisSendCounter.incr() + track(ibis2SendLatency)( + ibis2Sender.send(channels, candidate, notificationScribe, ntabResponse)) + } + } + + } + }.flatten + } + + def loggedOutNotify( + candidate: PushCandidate + ): Future[IbisResponse] = { + val ibisResponse = { + track(loggedOutIbis2SendLatency)( + ibis2Sender.send(Seq(ChannelName.PushNtab), candidate, notificationScribe, None)) + } + ibisResponse + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.scala new file mode 100644 index 0000000000..c2f7291158 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.scala @@ -0,0 +1,273 @@ +package com.twitter.frigate.pushservice.take + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.logger.MRLogger +import com.twitter.frigate.common.ntab.InvalidNTABWriteRequestException +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.notificationservice.thriftscala._ +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.configapi.Param +import com.twitter.util.Future +import scala.util.control.NoStackTrace + +class NtabCopyIdNotFoundException(private val message: String) + extends Exception(message) + with NoStackTrace + +class InvalidNtabCopyIdException(private val message: String) + extends Exception(message) + with NoStackTrace + +object NotificationServiceSender { + + def generateSocialContextTextEntities( + ntabDisplayNamesAndIdsFut: Future[Seq[(String, Long)]], + otherCountFut: Future[Int] + ): Future[Seq[DisplayTextEntity]] = { + Future.join(ntabDisplayNamesAndIdsFut, otherCountFut).map { + case (namesWithIdInOrder, otherCount) => + val displays = namesWithIdInOrder.zipWithIndex.map { + case ((name, id), index) => + DisplayTextEntity( + name = "user" + s"${index + 1}", + value = TextValue.Text(name), + emphasis = true, + userId = Some(id) + ) + } ++ Seq( + DisplayTextEntity(name = "nameCount", value = TextValue.Number(namesWithIdInOrder.size)) + ) + + val otherDisplay = if (otherCount > 0) { + Some( + DisplayTextEntity( + name = "otherCount", + value = TextValue.Number(otherCount) + ) + ) + } else None + displays ++ otherDisplay + } + } + + def getDisplayTextEntityFromUser( + userOpt: Option[User], + fieldName: String, + isBold: Boolean + ): Option[DisplayTextEntity] = { + for { + user <- userOpt + profile <- user.profile + } yield { + DisplayTextEntity( + name = fieldName, + value = TextValue.Text(profile.name), + emphasis = isBold, + userId = Some(user.id) + ) + } + } + + def getDisplayTextEntityFromUser( + user: Future[Option[User]], + fieldName: String, + isBold: Boolean + ): Future[Option[DisplayTextEntity]] = { + user.map { getDisplayTextEntityFromUser(_, fieldName, isBold) } + } +} + +case class NotificationServiceRequest( + candidate: PushCandidate, + impressionId: String, + isBadgeUpdate: Boolean, + overrideId: Option[String] = None) + +class NotificationServiceSender( + send: (Target, CreateGenericNotificationRequest) => Future[CreateGenericNotificationResponse], + enableWritesParam: Param[Boolean], + enableForEmployeesParam: Param[Boolean], + enableForEveryoneParam: Param[Boolean] +)( + implicit globalStats: StatsReceiver) + extends ReadableStore[NotificationServiceRequest, CreateGenericNotificationResponse] { + + val log = MRLogger(this.getClass.getName) + + val stats = globalStats.scope("NotificationServiceSender") + val requestEmpty = stats.scope("request_empty") + val requestNonEmpty = stats.counter("request_non_empty") + + val requestBadgeCount = stats.counter("request_badge_count") + + val successfulWrite = stats.counter("successful_write") + val successfulWriteScope = stats.scope("successful_write") + val failedWriteScope = stats.scope("failed_write") + val gotNonSuccessResponse = stats.counter("got_non_success_response") + val gotEmptyResponse = stats.counter("got_empty_response") + val deciderTurnedOffResponse = stats.scope("decider_turned_off_response") + + val disabledByDeciderForCandidate = stats.scope("model/candidate").counter("disabled_by_decider") + val sentToAlphaUserForCandidate = + stats.scope("model/candidate").counter("send_to_employee_or_team") + val sentToNonBucketedUserForCandidate = + stats.scope("model/candidate").counter("send_to_non_bucketed_decidered_user") + val noSendForCandidate = stats.scope("model/candidate").counter("no_send") + + val ineligibleUsersForCandidate = stats.scope("model/candidate").counter("ineligible_users") + + val darkWriteRequestsForCandidate = stats.scope("model/candidate").counter("dark_write_traffic") + + val heavyUserForCandidateCounter = stats.scope("model/candidate").counter("target_heavy") + val nonHeavyUserForCandidateCounter = stats.scope("model/candidate").counter("target_non_heavy") + + val skipWritingToNTAB = stats.counter("skip_writing_to_ntab") + + val ntabWriteDisabledForCandidate = stats.scope("model/candidate").counter("ntab_write_disabled") + + val ntabOverrideEnabledForCandidate = stats.scope("model/candidate").counter("override_enabled") + val ntabTTLForCandidate = stats.scope("model/candidate").counter("ttl_enabled") + + override def get( + notifRequest: NotificationServiceRequest + ): Future[Option[CreateGenericNotificationResponse]] = { + notifRequest.candidate.target.deviceInfo.flatMap { deviceInfoOpt => + val disableWritingToNtab = + notifRequest.candidate.target.params(PushParams.DisableWritingToNTAB) + + if (disableWritingToNtab) { + skipWritingToNTAB.incr() + Future.None + } else { + if (notifRequest.overrideId.nonEmpty) { ntabOverrideEnabledForCandidate.incr() } + Future + .join( + notifRequest.candidate.ntabRequest, + ntabWritesEnabledForCandidate(notifRequest.candidate)).flatMap { + case (Some(ntabRequest), ntabWritesEnabled) if ntabWritesEnabled => + if (ntabRequest.expiryTimeMillis.nonEmpty) { ntabTTLForCandidate.incr() } + sendNTabRequest( + ntabRequest, + notifRequest.candidate.target, + notifRequest.isBadgeUpdate, + notifRequest.candidate.commonRecType, + isFromCandidate = true, + overrideId = notifRequest.overrideId + ) + case (Some(_), ntabWritesEnabled) if !ntabWritesEnabled => + ntabWriteDisabledForCandidate.incr() + Future.None + case (None, ntabWritesEnabled) => + if (!ntabWritesEnabled) ntabWriteDisabledForCandidate.incr() + requestEmpty.counter(s"candidate_${notifRequest.candidate.commonRecType}").incr() + Future.None + } + } + } + } + + private def sendNTabRequest( + genericNotificationRequest: CreateGenericNotificationRequest, + target: Target, + isBadgeUpdate: Boolean, + crt: CommonRecommendationType, + isFromCandidate: Boolean, + overrideId: Option[String] + ): Future[Option[CreateGenericNotificationResponse]] = { + requestNonEmpty.incr() + val notifSvcReq = + genericNotificationRequest.copy( + sendBadgeCountUpdate = isBadgeUpdate, + overrideId = overrideId + ) + requestBadgeCount.incr() + send(target, notifSvcReq) + .map { response => + if (response.responseType.equals(CreateGenericNotificationResponseType.DecideredOff)) { + deciderTurnedOffResponse.counter(s"$crt").incr() + deciderTurnedOffResponse.counter(s"${genericNotificationRequest.genericType}").incr() + throw InvalidNTABWriteRequestException("Decider is turned off") + } else { + Some(response) + } + } + .onFailure { ex => + stats.counter(s"error_${ex.getClass.getCanonicalName}").incr() + failedWriteScope.counter(s"${crt}").incr() + log + .error( + ex, + s"NTAB failure $notifSvcReq" + ) + } + .onSuccess { + case Some(response) => + successfulWrite.incr() + val successfulWriteScopeString = if (isFromCandidate) "model/candidate" else "envelope" + successfulWriteScope.scope(successfulWriteScopeString).counter(s"$crt").incr() + if (response.responseType != CreateGenericNotificationResponseType.Success) { + gotNonSuccessResponse.incr() + log.warning(s"NTAB dropped $notifSvcReq with response $response") + } + + case _ => + gotEmptyResponse.incr() + } + } + + private def ntabWritesEnabledForCandidate(cand: PushCandidate): Future[Boolean] = { + if (!cand.target.params(enableWritesParam)) { + disabledByDeciderForCandidate.incr() + Future.False + } else { + Future + .join( + cand.target.isAnEmployee, + cand.target.isInNotificationsServiceWhitelist, + cand.target.isTeamMember + ) + .flatMap { + case (isEmployee, isInNotificationsServiceWhitelist, isTeamMember) => + cand.target.deviceInfo.flatMap { deviceInfoOpt => + deviceInfoOpt + .map { deviceInfo => + cand.target.isHeavyUserState.map { isHeavyUser => + val isAlphaTester = (isEmployee && cand.target + .params(enableForEmployeesParam)) || isInNotificationsServiceWhitelist || isTeamMember + if (cand.target.isDarkWrite) { + stats + .scope("model/candidate").counter( + s"dark_write_${cand.commonRecType}").incr() + darkWriteRequestsForCandidate.incr() + false + } else if (isAlphaTester || deviceInfo.isMRinNTabEligible + || cand.target.insertMagicrecsIntoNTabForNonPushableUsers) { + if (isHeavyUser) heavyUserForCandidateCounter.incr() + else nonHeavyUserForCandidateCounter.incr() + + val enabledForDesiredUsers = cand.target.params(enableForEveryoneParam) + if (isAlphaTester) { + sentToAlphaUserForCandidate.incr() + true + } else if (enabledForDesiredUsers) { + sentToNonBucketedUserForCandidate.incr() + true + } else { + noSendForCandidate.incr() + false + } + } else { + ineligibleUsersForCandidate.incr() + false + } + } + }.getOrElse(Future.False) + } + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.scala new file mode 100644 index 0000000000..feb65dffe9 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.scala @@ -0,0 +1,86 @@ +package com.twitter.frigate.pushservice.take + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.Invalid +import com.twitter.frigate.common.base.OK +import com.twitter.frigate.common.base.Response +import com.twitter.frigate.common.base.Result +import com.twitter.frigate.common.util.NotificationScribeUtil +import com.twitter.frigate.common.util.PushServiceUtil +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.thriftscala.PushResponse +import com.twitter.frigate.pushservice.thriftscala.PushStatus +import com.twitter.util.Future + +class SendHandlerNotifier( + candidateNotifier: CandidateNotifier, + private val statsReceiver: StatsReceiver) { + + val missingResponseCounter = statsReceiver.counter("missing_response") + val filteredResponseCounter = statsReceiver.counter("filtered") + + /** + * + * @param isScribeInfoRequired: [[Boolean]] to indicate if scribe info is required + * @param candidate: [[PushCandidate]] to build the scribe data from + * @return: scribe response string + */ + private def scribeInfoForResponse( + isScribeInfoRequired: Boolean, + candidate: PushCandidate + ): Future[Option[String]] = { + if (isScribeInfoRequired) { + candidate.scribeData().map { scribedInfo => + Some(NotificationScribeUtil.convertToJsonString(scribedInfo)) + } + } else Future.None + } + + /** + * + * @param response: Candidate validation response + * @param responseWithScribedInfo: boolean indicating if scribe data is expected in push response + * @return: [[PushResponse]] containing final result of send request for [[com.twitter.frigate.pushservice.thriftscala.PushRequest]] + */ + final def checkResponseAndNotify( + response: Response[PushCandidate, Result], + responseWithScribedInfo: Boolean + ): Future[PushResponse] = { + + response match { + case Response(OK, processedCandidates) => + val (validCandidates, invalidCandidates) = processedCandidates.partition(_.result == OK) + validCandidates.headOption match { + case Some(candidateResult) => + val scribeInfo = + scribeInfoForResponse(responseWithScribedInfo, candidateResult.candidate) + scribeInfo.flatMap { scribedData => + val response: Future[PushResponse] = + candidateNotifier.notify(candidateResult.candidate) + response.map(_.copy(notifScribe = scribedData)) + } + + case None => + invalidCandidates.headOption match { + case Some(candidateResult) => + filteredResponseCounter.incr() + val response = candidateResult.result match { + case Invalid(reason) => PushResponse(PushStatus.Filtered, filteredBy = reason) + case _ => PushResponse(PushStatus.Filtered, filteredBy = Some("unknown")) + } + + val scribeInfo = + scribeInfoForResponse(responseWithScribedInfo, candidateResult.candidate) + scribeInfo.map(scribeData => response.copy(notifScribe = scribeData)) + + case None => + missingResponseCounter.incr() + PushServiceUtil.FilteredPushResponseFut + } + } + + case Response(Invalid(reason), _) => + throw new IllegalStateException(s"Unexpected target filtering in SendHandler: $reason") + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.scala new file mode 100644 index 0000000000..ee85ba5902 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.scala @@ -0,0 +1,83 @@ +package com.twitter.frigate.pushservice.take.candidate_validator + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.logger.MRLogger +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.take.predicates.TakeCommonPredicates +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.hermit.predicate.ConcurrentPredicate +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.hermit.predicate.Predicate +import com.twitter.hermit.predicate.SequentialPredicate +import com.twitter.util.Future + +trait CandidateValidator extends TakeCommonPredicates { + + override implicit val statsReceiver: StatsReceiver = config.statsReceiver + + protected val log = MRLogger("CandidateValidator") + + private lazy val skipFiltersCounter = statsReceiver.counter("enable_skip_filters") + private lazy val emailUserSkipFiltersCounter = + statsReceiver.counter("email_user_enable_skip_filters") + private lazy val enablePredicatesCounter = statsReceiver.counter("enable_predicates") + + protected def enabledPredicates[C <: PushCandidate]( + candidate: C, + predicates: List[NamedPredicate[C]] + ): List[NamedPredicate[C]] = { + val target = candidate.target + val skipFilters: Boolean = + target.pushContext.flatMap(_.skipFilters).getOrElse(false) || target.params( + PushFeatureSwitchParams.SkipPostRankingFilters) + + if (skipFilters) { + skipFiltersCounter.incr() + if (target.isEmailUser) emailUserSkipFiltersCounter.incr() + + val predicatesToEnable = target.pushContext.flatMap(_.predicatesToEnable).getOrElse(Nil) + if (predicatesToEnable.nonEmpty) enablePredicatesCounter.incr() + + // if we skip predicates on pushContext, only enable the explicitly specified predicates + predicates.filter(predicatesToEnable.contains) + } else predicates + } + + protected def executeSequentialPredicates[C <: PushCandidate]( + candidate: C, + predicates: List[NamedPredicate[C]] + ): Future[Option[Predicate[C]]] = { + val predicatesEnabled = enabledPredicates(candidate, predicates) + val sequentialPredicate = new SequentialPredicate(predicatesEnabled) + + sequentialPredicate.track(Seq(candidate)).map(_.head) + } + + protected def executeConcurrentPredicates[C <: PushCandidate]( + candidate: C, + predicates: List[NamedPredicate[C]] + ): Future[List[Predicate[C]]] = { + val predicatesEnabled = enabledPredicates(candidate, predicates) + val concurrentPredicate: ConcurrentPredicate[C] = new ConcurrentPredicate[C](predicatesEnabled) + concurrentPredicate.track(Seq(candidate)).map(_.head) + } + + protected val candidatePredicatesMap: Map[CommonRecommendationType, List[ + NamedPredicate[_ <: PushCandidate] + ]] + + protected def getCRTPredicates[C <: PushCandidate]( + CRT: CommonRecommendationType + ): List[NamedPredicate[C]] = { + candidatePredicatesMap.get(CRT) match { + case Some(predicates) => + predicates.asInstanceOf[List[NamedPredicate[C]]] + case _ => + throw new IllegalStateException( + s"Unknown CommonRecommendationType for Predicates: ${CRT.name}") + } + } + + def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.scala new file mode 100644 index 0000000000..ecc99cc9e8 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.scala @@ -0,0 +1,27 @@ +package com.twitter.frigate.pushservice.take.candidate_validator + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.take.predicates.candidate_map.CandidatePredicatesMap +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +class RFPHCandidateValidator(override val config: Config) extends CandidateValidator { + private val rFPHCandidateValidatorStats = statsReceiver.scope(this.getClass.getSimpleName) + private val concurrentPredicateCount = rFPHCandidateValidatorStats.counter("concurrent") + private val sequentialPredicateCount = rFPHCandidateValidatorStats.counter("sequential") + + override protected val candidatePredicatesMap = CandidatePredicatesMap(config) + + override def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] = { + val candidatePredicates = getCRTPredicates(candidate.commonRecType) + val predicates = rfphPrePredicates ++ candidatePredicates ++ postPredicates + if (candidate.target.isEmailUser) { + concurrentPredicateCount.incr() + executeConcurrentPredicates(candidate, predicates).map(_.headOption) + } else { + sequentialPredicateCount.incr() + executeSequentialPredicates(candidate, predicates) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.scala new file mode 100644 index 0000000000..096e9f1027 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.scala @@ -0,0 +1,26 @@ +package com.twitter.frigate.pushservice.take.candidate_validator + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.take.predicates.candidate_map.SendHandlerCandidatePredicatesMap +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +class SendHandlerPostCandidateValidator(override val config: Config) extends CandidateValidator { + + override protected val candidatePredicatesMap = + SendHandlerCandidatePredicatesMap.postCandidatePredicates(config) + + private val sendHandlerPostCandidateValidatorStats = + statsReceiver.counter("sendHandlerPostCandidateValidator_stats") + + override def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] = { + val candidatePredicates = getCRTPredicates(candidate.commonRecType) + val predicates = candidatePredicates ++ postPredicates + + sendHandlerPostCandidateValidatorStats.incr() + + executeConcurrentPredicates(candidate, predicates) + .map(_.headOption) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.scala new file mode 100644 index 0000000000..eb0293017b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.scala @@ -0,0 +1,24 @@ +package com.twitter.frigate.pushservice.take.candidate_validator + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.take.predicates.candidate_map.SendHandlerCandidatePredicatesMap +import com.twitter.hermit.predicate.Predicate +import com.twitter.util.Future + +class SendHandlerPreCandidateValidator(override val config: Config) extends CandidateValidator { + + override protected val candidatePredicatesMap = + SendHandlerCandidatePredicatesMap.preCandidatePredicates(config) + + private val sendHandlerPreCandidateValidatorStats = + statsReceiver.counter("sendHandlerPreCandidateValidator_stats") + + override def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] = { + val candidatePredicates = getCRTPredicates(candidate.commonRecType) + val predicates = sendHandlerPrePredicates ++ candidatePredicates + + sendHandlerPreCandidateValidatorStats.incr() + executeSequentialPredicates(candidate, predicates) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.scala new file mode 100644 index 0000000000..a278ef2371 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.scala @@ -0,0 +1,24 @@ +package com.twitter.frigate.pushservice.take + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.thriftscala.ChannelName +import com.twitter.util.Future +import java.util.concurrent.ConcurrentHashMap +import scala.collection.concurrent +import scala.collection.convert.decorateAsScala._ + +/** + * A class to save all the channel related information + */ +trait ChannelForCandidate { + self: PushCandidate => + + // Cache of channel selection result + private[this] val selectedChannels: concurrent.Map[String, Future[Seq[ChannelName]]] = + new ConcurrentHashMap[String, Future[Seq[ChannelName]]]().asScala + + // Returns the channel information from all ChannelSelectors. + def getChannels(): Future[Seq[ChannelName]] = { + Future.collect(selectedChannels.values.toSeq).map { c => c.flatten } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.scala new file mode 100644 index 0000000000..62378a4bc2 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.scala @@ -0,0 +1,15 @@ +package com.twitter.frigate.pushservice.take + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.thriftscala.ChannelName +import com.twitter.util.Future + +abstract class ChannelSelector { + + // Returns a map of channel name, and the candidates that can be sent on that channel. + def selectChannel( + candidate: PushCandidate + ): Future[Seq[ChannelName]] + + def getSelectorName(): String +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.scala new file mode 100644 index 0000000000..e999da9bec --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.scala @@ -0,0 +1,21 @@ +package com.twitter.frigate.pushservice.take + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.thriftscala.ChannelName +import com.twitter.util.Future + +class NtabOnlyChannelSelector extends ChannelSelector { + val SELECTOR_NAME = "NtabOnlyChannelSelector" + + def getSelectorName(): String = SELECTOR_NAME + + // Returns a map of channel name, and the candidates that can be sent on that channel + def selectChannel( + candidate: PushCandidate + ): Future[Seq[ChannelName]] = { + // Check candidate channel eligible (based on setting, push cap etc + // Decide which candidate can be sent on what channel + val channelName: Future[ChannelName] = Future.value(ChannelName.PushNtab) + channelName.map(channel => Seq(channel)) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.scala new file mode 100644 index 0000000000..2bdf412ac8 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.scala @@ -0,0 +1,37 @@ +package com.twitter.frigate.pushservice.take.history + +import com.twitter.eventbus.client.EventBusPublisher +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.util.NotificationScribeUtil +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.scribe.thriftscala.NotificationScribe +import com.twitter.frigate.thriftscala.FrigateNotification + +class EventBusWriter( + eventBusPublisher: EventBusPublisher[NotificationScribe], + stats: StatsReceiver) { + private def writeSendEventToEventBus( + target: PushTypes.Target, + notificationScribe: NotificationScribe + ): Unit = { + if (target.params(PushParams.EnablePushSendEventBus)) { + val result = eventBusPublisher.publish(notificationScribe) + result.onFailure { _ => stats.counter("push_send_eventbus_failure").incr() } + } + } + + def writeToEventBus( + candidate: PushCandidate, + frigateNotificationForPersistence: FrigateNotification + ): Unit = { + val notificationScribe = NotificationScribeUtil.getNotificationScribe( + targetId = candidate.target.targetId, + impressionId = candidate.impressionId, + frigateNotification = frigateNotificationForPersistence, + createdAt = candidate.createdAt + ) + writeSendEventToEventBus(candidate.target, notificationScribe) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.scala new file mode 100644 index 0000000000..ca9fe31bcb --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.scala @@ -0,0 +1,49 @@ +package com.twitter.frigate.pushservice.take.history + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.history.HistoryStoreKeyContext +import com.twitter.frigate.common.history.PushServiceHistoryStore +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.conversions.DurationOps._ + +class HistoryWriter(historyStore: PushServiceHistoryStore, stats: StatsReceiver) { + private lazy val historyWriterStats = stats.scope(this.getClass.getSimpleName) + private lazy val historyWriteCounter = historyWriterStats.counter("history_write_num") + private lazy val loggedOutHistoryWriteCounter = + historyWriterStats.counter("logged_out_history_write_num") + + private def writeTtlForHistory(candidate: PushCandidate): Duration = { + if (candidate.target.isLoggedOutUser) { + 60.days + } else if (RecTypes.isTweetType(candidate.commonRecType)) { + candidate.target.params(PushFeatureSwitchParams.FrigateHistoryTweetNotificationWriteTtl) + } else candidate.target.params(PushFeatureSwitchParams.FrigateHistoryOtherNotificationWriteTtl) + } + + def writeSendToHistory( + candidate: PushCandidate, + frigateNotificationForPersistence: FrigateNotification + ): Future[Unit] = { + val historyStoreKeyContext = HistoryStoreKeyContext( + candidate.target.targetId, + candidate.target.pushContext.flatMap(_.useMemcacheForHistory).getOrElse(false) + ) + if (candidate.target.isLoggedOutUser) { + loggedOutHistoryWriteCounter.incr() + } else { + historyWriteCounter.incr() + } + historyStore + .put( + historyStoreKeyContext, + candidate.createdAt, + frigateNotificationForPersistence, + Some(writeTtlForHistory(candidate)) + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.scala new file mode 100644 index 0000000000..99719af99c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.scala @@ -0,0 +1,7 @@ +package com.twitter.frigate.pushservice.take.predicates +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.predicate.NamedPredicate + +trait BasicRFPHPredicates[C <: PushCandidate] { + val predicates: List[NamedPredicate[C]] +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.scala new file mode 100644 index 0000000000..591a4df75b --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.scala @@ -0,0 +1,13 @@ +package com.twitter.frigate.pushservice.take.predicates + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.predicate.NamedPredicate + +trait BasicSendHandlerPredicates[C <: PushCandidate] { + + // specific predicates per candidate type before basic SendHandler predicates + val preCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty + + // specific predicates per candidate type after basic SendHandler predicates, could be empty + val postCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.scala new file mode 100644 index 0000000000..0750abd6e6 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.scala @@ -0,0 +1,104 @@ +package com.twitter.frigate.pushservice.take.predicates + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.predicate.BqmlHealthModelPredicates +import com.twitter.frigate.pushservice.predicate.BqmlQualityModelPredicates +import com.twitter.frigate.pushservice.predicate.HealthPredicates +import com.twitter.frigate.pushservice.predicate.OONSpreadControlPredicate +import com.twitter.frigate.pushservice.predicate.OONTweetNegativeFeedbackBasedPredicate +import com.twitter.frigate.pushservice.predicate.OutOfNetworkCandidatesQualityPredicates +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.PNegMultimodalPredicates +import com.twitter.frigate.pushservice.predicate.TargetEngagementPredicate +import com.twitter.frigate.pushservice.predicate.TweetEngagementRatioPredicate +import com.twitter.frigate.pushservice.predicate.TweetLanguagePredicate +import com.twitter.frigate.pushservice.predicate.TweetWithheldContentPredicate + +trait BasicTweetPredicates { + + def config: Config + + implicit def statsReceiver: StatsReceiver + + final lazy val basicTweetPredicates = + List( + HealthPredicates.sensitiveMediaCategoryPredicate(), + HealthPredicates.profanityPredicate(), + PredicatesForCandidate.disableOutNetworkTweetPredicate(config.edgeStore), + TweetEngagementRatioPredicate.QTtoNtabClickBasedPredicate(), + TweetLanguagePredicate.oonTweeetLanguageMatch(), + HealthPredicates.userHealthSignalsPredicate(config.userHealthSignalStore), + HealthPredicates.authorSensitiveMediaPredicate(config.producerMediaRepresentationStore), + HealthPredicates.authorProfileBasedPredicate(), + PNegMultimodalPredicates.healthSignalScorePNegMultimodalPredicate( + config.tweetHealthScoreStore), + BqmlHealthModelPredicates.healthModelOonPredicate( + config.filteringModelScorer, + config.producerMediaRepresentationStore, + config.userHealthSignalStore, + config.tweetHealthScoreStore), + BqmlQualityModelPredicates.BqmlQualityModelOonPredicate(config.filteringModelScorer), + HealthPredicates.tweetHealthSignalScorePredicate(config.tweetHealthScoreStore), + HealthPredicates + .tweetHealthSignalScorePredicate(config.tweetHealthScoreStore, applyToQuoteTweet = true), + PredicatesForCandidate.nullCastF1ProtectedExperientPredicate( + config.cachedTweetyPieStoreV2 + ), + OONTweetNegativeFeedbackBasedPredicate.ntabDislikeBasedPredicate(), + OONSpreadControlPredicate.oonTweetSpreadControlPredicate(), + OONSpreadControlPredicate.oonAuthorSpreadControlPredicate(), + HealthPredicates.healthSignalScoreMultilingualPnsfwTweetTextPredicate( + config.tweetHealthScoreStore), + PredicatesForCandidate + .recommendedTweetAuthorAcceptableToTargetUser(config.edgeStore), + HealthPredicates.healthSignalScorePnsfwTweetTextPredicate(config.tweetHealthScoreStore), + HealthPredicates.healthSignalScoreSpammyTweetPredicate(config.tweetHealthScoreStore), + OutOfNetworkCandidatesQualityPredicates.NegativeKeywordsPredicate( + config.postRankingFeatureStoreClient), + PredicatesForCandidate.authorNotBeingDeviceFollowed(config.edgeStore), + TweetWithheldContentPredicate(), + PredicatesForCandidate.noOptoutFreeFormInterestPredicate, + PredicatesForCandidate.disableInNetworkTweetPredicate(config.edgeStore), + TweetEngagementRatioPredicate.TweetReplyLikeRatioPredicate(), + TargetEngagementPredicate( + config.userTweetPerspectiveStore, + defaultForMissing = true + ), + ) +} + +/** + * This trait is a new version of BasicTweetPredicates + * Difference from old version is that basicTweetPredicates are different + * basicTweetPredicates here don't include Social Graph Service related predicates + */ +trait BasicTweetPredicatesWithoutSGSPredicates { + + def config: Config + + implicit def statsReceiver: StatsReceiver + + final lazy val basicTweetPredicates = { + List( + HealthPredicates.healthSignalScoreSpammyTweetPredicate(config.tweetHealthScoreStore), + PredicatesForCandidate.nullCastF1ProtectedExperientPredicate( + config.cachedTweetyPieStoreV2 + ), + TweetWithheldContentPredicate(), + TargetEngagementPredicate( + config.userTweetPerspectiveStore, + defaultForMissing = true + ), + PredicatesForCandidate.noOptoutFreeFormInterestPredicate, + HealthPredicates.userHealthSignalsPredicate(config.userHealthSignalStore), + HealthPredicates.tweetHealthSignalScorePredicate(config.tweetHealthScoreStore), + BqmlQualityModelPredicates.BqmlQualityModelOonPredicate(config.filteringModelScorer), + BqmlHealthModelPredicates.healthModelOonPredicate( + config.filteringModelScorer, + config.producerMediaRepresentationStore, + config.userHealthSignalStore, + config.tweetHealthScoreStore), + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.scala new file mode 100644 index 0000000000..7d660632ae --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.scala @@ -0,0 +1,41 @@ +package com.twitter.frigate.pushservice.take.predicates + +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.base.TweetDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.hermit.predicate.NamedPredicate + +trait BasicTweetPredicatesForRFPH[C <: PushCandidate with TweetCandidate with TweetDetails] + extends BasicTweetPredicates + with BasicRFPHPredicates[C] { + + // specific predicates per candidate type before basic tweet predicates + def preCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty + + // specific predicates per candidate type after basic tweet predicates + def postCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty + + override lazy val predicates: List[NamedPredicate[C]] = + preCandidateSpecificPredicates ++ basicTweetPredicates ++ postCandidateSpecificPredicates +} + +/** + * This trait is a new version of BasicTweetPredicatesForRFPH + * Difference from old version is that basicTweetPredicates are different + * basicTweetPredicates here don't include Social Graph Service related predicates + */ +trait BasicTweetPredicatesForRFPHWithoutSGSPredicates[ + C <: PushCandidate with TweetCandidate with TweetDetails] + extends BasicTweetPredicatesWithoutSGSPredicates + with BasicRFPHPredicates[C] { + + // specific predicates per candidate type before basic tweet predicates + def preCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty + + // specific predicates per candidate type after basic tweet predicates + def postCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty + + override lazy val predicates: List[NamedPredicate[C]] = + preCandidateSpecificPredicates ++ basicTweetPredicates ++ postCandidateSpecificPredicates + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.scala new file mode 100644 index 0000000000..e85dc95f0d --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.scala @@ -0,0 +1,16 @@ +package com.twitter.frigate.pushservice.take.predicates + +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.base.TweetDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.hermit.predicate.NamedPredicate + +trait OutOfNetworkTweetPredicates[C <: PushCandidate with TweetCandidate with TweetDetails] + extends BasicTweetPredicatesForRFPH[C] { + + override lazy val preCandidateSpecificPredicates: List[NamedPredicate[C]] = + List( + PredicatesForCandidate.authorNotBeingFollowed(config.edgeStore) + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.scala new file mode 100644 index 0000000000..e921f3fcfc --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.scala @@ -0,0 +1,36 @@ +package com.twitter.frigate.pushservice.take.predicates + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.pushservice.predicate.CrtDeciderPredicate +import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate +import com.twitter.frigate.pushservice.predicate.ScarecrowPredicate +import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicate +import com.twitter.hermit.predicate.NamedPredicate + +trait TakeCommonPredicates { + def config: Config + + implicit def statsReceiver: StatsReceiver + + lazy val rfphPrePredicates: List[NamedPredicate[PushCandidate]] = List( + CrtDeciderPredicate(config.decider), + PredicatesForCandidate.isChannelValidPredicate, + ) + + lazy val sendHandlerPrePredicates: List[NamedPredicate[PushCandidate]] = List( + CrtDeciderPredicate(config.decider), + PredicatesForCandidate.enableSendHandlerCandidates, + PredicatesForCandidate.mrWebHoldbackPredicate, + PredicatesForCandidate.targetUserExists, + PredicatesForCandidate.authorInSocialContext, + PredicatesForCandidate.recommendedTweetIsAuthoredBySelf, + PredicatesForCandidate.selfInSocialContext, + NtabCaretClickFatiguePredicate() + ) + + lazy val postPredicates: List[NamedPredicate[PushCandidate]] = List( + ScarecrowPredicate(config.scarecrowCheckEventStore) + ) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.scala new file mode 100644 index 0000000000..aa4642cea9 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.scala @@ -0,0 +1,75 @@ +package com.twitter.frigate.pushservice.take.predicates.candidate_map + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model._ +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.CommonRecommendationType._ +import com.twitter.hermit.predicate.NamedPredicate + +object CandidatePredicatesMap { + + def apply( + implicit config: Config + ): Map[CommonRecommendationType, List[NamedPredicate[_ <: PushCandidate]]] = { + + val trendTweetCandidatePredicates = TrendTweetPredicates(config).predicates + val tripTweetCandidatePredicates = TripTweetCandidatePredicates(config).predicates + val f1TweetCandidatePredicates = F1TweetCandidatePredicates(config).predicates + val oonTweetCandidatePredicates = OutOfNetworkTweetCandidatePredicates(config).predicates + val tweetActionCandidatePredicates = TweetActionCandidatePredicates(config).predicates + val topicProofTweetCandidatePredicates = TopicProofTweetCandidatePredicates(config).predicates + val addressBookPushPredicates = AddressBookPushCandidatePredicates(config).predicates + val completeOnboardingPushPredicates = CompleteOnboardingPushCandidatePredicates( + config).predicates + val popGeoTweetCandidatePredicate = PopGeoTweetCandidatePredicates(config).predicates + val topTweetImpressionsCandidatePredicates = TopTweetImpressionsPushCandidatePredicates( + config).predicates + val listCandidatePredicates = ListRecommendationPredicates(config).predicates + val subscribedSearchTweetCandidatePredicates = SubscribedSearchTweetCandidatePredicates( + config).predicates + + Map( + F1FirstdegreeTweet -> f1TweetCandidatePredicates, + F1FirstdegreePhoto -> f1TweetCandidatePredicates, + F1FirstdegreeVideo -> f1TweetCandidatePredicates, + ElasticTimelineTweet -> oonTweetCandidatePredicates, + ElasticTimelinePhoto -> oonTweetCandidatePredicates, + ElasticTimelineVideo -> oonTweetCandidatePredicates, + TwistlyTweet -> oonTweetCandidatePredicates, + TwistlyPhoto -> oonTweetCandidatePredicates, + TwistlyVideo -> oonTweetCandidatePredicates, + ExploreVideoTweet -> oonTweetCandidatePredicates, + UserInterestinTweet -> oonTweetCandidatePredicates, + UserInterestinPhoto -> oonTweetCandidatePredicates, + UserInterestinVideo -> oonTweetCandidatePredicates, + PastEmailEngagementTweet -> oonTweetCandidatePredicates, + PastEmailEngagementPhoto -> oonTweetCandidatePredicates, + PastEmailEngagementVideo -> oonTweetCandidatePredicates, + TagSpaceTweet -> oonTweetCandidatePredicates, + TwhinTweet -> oonTweetCandidatePredicates, + FrsTweet -> oonTweetCandidatePredicates, + MrModelingBasedTweet -> oonTweetCandidatePredicates, + TrendTweet -> trendTweetCandidatePredicates, + ReverseAddressbookTweet -> oonTweetCandidatePredicates, + ForwardAddressbookTweet -> oonTweetCandidatePredicates, + TripGeoTweet -> oonTweetCandidatePredicates, + TripHqTweet -> tripTweetCandidatePredicates, + DetopicTweet -> oonTweetCandidatePredicates, + CrowdSearchTweet -> oonTweetCandidatePredicates, + TweetFavorite -> tweetActionCandidatePredicates, + TweetFavoritePhoto -> tweetActionCandidatePredicates, + TweetFavoriteVideo -> tweetActionCandidatePredicates, + TweetRetweet -> tweetActionCandidatePredicates, + TweetRetweetPhoto -> tweetActionCandidatePredicates, + TweetRetweetVideo -> tweetActionCandidatePredicates, + TopicProofTweet -> topicProofTweetCandidatePredicates, + SubscribedSearch -> subscribedSearchTweetCandidatePredicates, + AddressBookUploadPush -> addressBookPushPredicates, + CompleteOnboardingPush -> completeOnboardingPushPredicates, + List -> listCandidatePredicates, + GeoPopTweet -> popGeoTweetCandidatePredicate, + TweetImpressions -> topTweetImpressionsCandidatePredicates + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.scala new file mode 100644 index 0000000000..e37a910449 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.scala @@ -0,0 +1,78 @@ +package com.twitter.frigate.pushservice.take.predicates.candidate_map + +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model._ +import com.twitter.frigate.pushservice.config.Config +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.CommonRecommendationType._ +import com.twitter.hermit.predicate.NamedPredicate + +object SendHandlerCandidatePredicatesMap { + + def preCandidatePredicates( + implicit config: Config + ): Map[CommonRecommendationType, List[NamedPredicate[_ <: PushCandidate]]] = { + val magicFanoutNewsEventCandidatePredicates = + MagicFanoutNewsEventCandidatePredicates(config).preCandidateSpecificPredicates + + val scheduledSpaceSubscriberPredicates = ScheduledSpaceSubscriberCandidatePredicates( + config).preCandidateSpecificPredicates + + val scheduledSpaceSpeakerPredicates = ScheduledSpaceSpeakerCandidatePredicates( + config).preCandidateSpecificPredicates + + val magicFanoutSportsEventCandidatePredicates = + MagicFanoutSportsEventCandidatePredicates(config).preCandidateSpecificPredicates + + val magicFanoutProductLaunchPredicates = MagicFanoutProductLaunchPushCandidatePredicates( + config).preCandidateSpecificPredicates + + val creatorSubscriptionFanoutPredicates = MagicFanouCreatorSubscriptionEventPushPredicates( + config).preCandidateSpecificPredicates + + val newCreatorFanoutPredicates = MagicFanoutNewCreatorEventPushPredicates( + config).preCandidateSpecificPredicates + + Map( + MagicFanoutNewsEvent -> magicFanoutNewsEventCandidatePredicates, + ScheduledSpaceSubscriber -> scheduledSpaceSubscriberPredicates, + ScheduledSpaceSpeaker -> scheduledSpaceSpeakerPredicates, + MagicFanoutSportsEvent -> magicFanoutSportsEventCandidatePredicates, + MagicFanoutProductLaunch -> magicFanoutProductLaunchPredicates, + NewCreator -> newCreatorFanoutPredicates, + CreatorSubscriber -> creatorSubscriptionFanoutPredicates + ) + } + + def postCandidatePredicates( + implicit config: Config + ): Map[CommonRecommendationType, List[NamedPredicate[_ <: PushCandidate]]] = { + val magicFanoutNewsEventCandidatePredicates = + MagicFanoutNewsEventCandidatePredicates(config).postCandidateSpecificPredicates + + val scheduledSpaceSubscriberPredicates = ScheduledSpaceSubscriberCandidatePredicates( + config).postCandidateSpecificPredicates + + val scheduledSpaceSpeakerPredicates = ScheduledSpaceSpeakerCandidatePredicates( + config).postCandidateSpecificPredicates + + val magicFanoutSportsEventCandidatePredicates = + MagicFanoutSportsEventCandidatePredicates(config).postCandidateSpecificPredicates + val magicFanoutProductLaunchPredicates = MagicFanoutProductLaunchPushCandidatePredicates( + config).postCandidateSpecificPredicates + val creatorSubscriptionFanoutPredicates = MagicFanouCreatorSubscriptionEventPushPredicates( + config).postCandidateSpecificPredicates + val newCreatorFanoutPredicates = MagicFanoutNewCreatorEventPushPredicates( + config).postCandidateSpecificPredicates + + Map( + MagicFanoutNewsEvent -> magicFanoutNewsEventCandidatePredicates, + ScheduledSpaceSubscriber -> scheduledSpaceSubscriberPredicates, + ScheduledSpaceSpeaker -> scheduledSpaceSpeakerPredicates, + MagicFanoutSportsEvent -> magicFanoutSportsEventCandidatePredicates, + MagicFanoutProductLaunch -> magicFanoutProductLaunchPredicates, + NewCreator -> newCreatorFanoutPredicates, + CreatorSubscriber -> creatorSubscriptionFanoutPredicates + ) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.scala new file mode 100644 index 0000000000..a17d7fe1e9 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.scala @@ -0,0 +1,185 @@ +package com.twitter.frigate.pushservice.take.sender + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.common.base.TweetDetails +import com.twitter.frigate.common.store.IbisResponse +import com.twitter.frigate.common.store.InvalidConfiguration +import com.twitter.frigate.common.store.NoRequest +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} +import com.twitter.frigate.pushservice.store.Ibis2Store +import com.twitter.frigate.pushservice.store.TweetTranslationStore +import com.twitter.frigate.pushservice.util.CopyUtil +import com.twitter.frigate.pushservice.util.FunctionalUtil +import com.twitter.frigate.pushservice.util.InlineActionUtil +import com.twitter.frigate.pushservice.util.OverrideNotificationUtil +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.frigate.scribe.thriftscala.NotificationScribe +import com.twitter.frigate.thriftscala.ChannelName +import com.twitter.frigate.thriftscala.NotificationDisplayLocation +import com.twitter.ibis2.service.thriftscala.Ibis2Request +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +class Ibis2Sender( + pushIbisV2Store: Ibis2Store, + tweetTranslationStore: ReadableStore[TweetTranslationStore.Key, TweetTranslationStore.Value], + statsReceiver: StatsReceiver) { + + private val stats = statsReceiver.scope(getClass.getSimpleName) + private val silentPushCounter = stats.counter("silent_push") + private val ibisSendFailureCounter = stats.scope("ibis_send_failure").counter("failures") + private val buggyAndroidReleaseCounter = stats.counter("is_buggy_android_release") + private val androidPrimaryCounter = stats.counter("android_primary_device") + private val addTranslationModelValuesCounter = stats.counter("with_translation_model_values") + private val patchNtabResponseEnabled = stats.scope("with_ntab_response") + private val noIbisPushStats = stats.counter("no_ibis_push") + + private def ibisSend( + candidate: PushCandidate, + translationModelValues: Option[Map[String, String]] = None, + ntabResponse: Option[CreateGenericNotificationResponse] = None + ): Future[IbisResponse] = { + if (candidate.frigateNotification.notificationDisplayLocation != NotificationDisplayLocation.PushToMobileDevice) { + Future.value(IbisResponse(InvalidConfiguration)) + } else { + candidate.ibis2Request.flatMap { + case Some(request) => + val requestWithTranslationMV = + addTranslationModelValues(request, translationModelValues) + val patchedIbisRequest = { + if (candidate.target.isLoggedOutUser) { + requestWithTranslationMV + } else { + patchNtabResponseToIbisRequest(requestWithTranslationMV, candidate, ntabResponse) + } + } + pushIbisV2Store.send(patchedIbisRequest, candidate) + case _ => + noIbisPushStats.incr() + Future.value(IbisResponse(sendStatus = NoRequest, ibis2Response = None)) + } + } + } + + def sendAsDarkWrite( + candidate: PushCandidate + ): Future[IbisResponse] = { + ibisSend(candidate) + } + + def send( + channels: Seq[ChannelName], + pushCandidate: PushCandidate, + notificationScribe: NotificationScribe => Unit, + ntabResponse: Option[CreateGenericNotificationResponse], + ): Future[IbisResponse] = pushCandidate.target.isSilentPush.flatMap { isSilentPush: Boolean => + if (isSilentPush) silentPushCounter.incr() + pushCandidate.target.deviceInfo.flatMap { deviceInfo => + if (deviceInfo.exists(_.isSim40AndroidVersion)) buggyAndroidReleaseCounter.incr() + if (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfo)) androidPrimaryCounter.incr() + Future + .join( + OverrideNotificationUtil + .getOverrideInfo(pushCandidate, stats), + CopyUtil.getCopyFeatures(pushCandidate, stats), + getTranslationModelValues(pushCandidate) + ).flatMap { + case (overrideInfoOpt, copyFeaturesMap, translationModelValues) => + ibisSend(pushCandidate, translationModelValues, ntabResponse) + .onSuccess { ibisResponse => + pushCandidate + .scribeData( + ibis2Response = ibisResponse.ibis2Response, + isSilentPush = isSilentPush, + overrideInfoOpt = overrideInfoOpt, + copyFeaturesList = copyFeaturesMap.keySet, + channels = channels + ).foreach(notificationScribe) + }.onFailure { _ => + pushCandidate + .scribeData(channels = channels).foreach { data => + ibisSendFailureCounter.incr() + notificationScribe(data) + } + } + } + } + } + + private def getTranslationModelValues( + candidate: PushCandidate + ): Future[Option[Map[String, String]]] = { + candidate match { + case tweetCandidate: TweetCandidate with TweetDetails => + val key = TweetTranslationStore.Key( + target = candidate.target, + tweetId = tweetCandidate.tweetId, + tweet = tweetCandidate.tweet, + crt = candidate.commonRecType + ) + + tweetTranslationStore + .get(key) + .map { + case Some(value) => + Some( + Map( + "translated_tweet_text" -> value.translatedTweetText, + "localized_source_language" -> value.localizedSourceLanguage + )) + case None => None + } + case _ => Future.None + } + } + + private def addTranslationModelValues( + ibisRequest: Ibis2Request, + translationModelValues: Option[Map[String, String]] + ): Ibis2Request = { + (translationModelValues, ibisRequest.modelValues) match { + case (Some(translationModelVal), Some(existingModelValues)) => + addTranslationModelValuesCounter.incr() + ibisRequest.copy(modelValues = Some(translationModelVal ++ existingModelValues)) + case (Some(translationModelVal), None) => + addTranslationModelValuesCounter.incr() + ibisRequest.copy(modelValues = Some(translationModelVal)) + case (None, _) => ibisRequest + } + } + + private def patchNtabResponseToIbisRequest( + ibis2Req: Ibis2Request, + candidate: PushCandidate, + ntabResponse: Option[CreateGenericNotificationResponse] + ): Ibis2Request = { + if (candidate.target.params(FS.EnableInlineFeedbackOnPush)) { + patchNtabResponseEnabled.counter().incr() + val dislikePosition = candidate.target.params(FS.InlineFeedbackSubstitutePosition) + val dislikeActionOption = ntabResponse + .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("ntab_response_exist"))) + .flatMap(response => InlineActionUtil.getDislikeInlineAction(candidate, response)) + .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("dislike_action_generated"))) + + // Only generate patch serialized inline action when original request has existing serialized_inline_actions_v2 + val patchedSerializedActionOption = ibis2Req.modelValues + .flatMap(model => model.get("serialized_inline_actions_v2")) + .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("inline_action_v2_exists"))) + .map(serialized => + InlineActionUtil + .patchInlineActionAtPosition(serialized, dislikeActionOption, dislikePosition)) + .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("patch_inline_action_generated"))) + + (ibis2Req.modelValues, patchedSerializedActionOption) match { + case (Some(existingModelValue), Some(patchedActionV2)) => + patchNtabResponseEnabled.scope("patch_applied").counter().incr() + ibis2Req.copy(modelValues = + Some(existingModelValue ++ Map("serialized_inline_actions_v2" -> patchedActionV2))) + case _ => ibis2Req + } + } else ibis2Req + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.scala new file mode 100644 index 0000000000..5019aa0404 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.scala @@ -0,0 +1,237 @@ +package com.twitter.frigate.pushservice.take.sender + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.history.History +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.ibis.PushOverrideInfo +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} +import com.twitter.frigate.pushservice.take.NotificationServiceRequest +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.hermit.store.common.ReadableWritableStore +import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse +import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest +import com.twitter.notificationservice.thriftscala.GenericNotificationKey +import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +object OverrideCandidate extends Enumeration { + val One: String = "overrideEntry1" +} + +class NtabSender( + notificationServiceSender: ReadableStore[ + NotificationServiceRequest, + CreateGenericNotificationResponse + ], + nTabHistoryStore: ReadableWritableStore[(Long, String), GenericNotificationOverrideKey], + nTabDelete: DeleteGenericNotificationRequest => Future[Unit], + nTabDeleteTimeline: DeleteCurrentTimelineForUserRequest => Future[Unit] +)( + implicit statsReceiver: StatsReceiver) { + + private[this] val nTabDeleteRequests = statsReceiver.counter("ntab_delete_request") + private[this] val nTabDeleteTimelineRequests = + statsReceiver.counter("ntab_delete_timeline_request") + private[this] val ntabOverrideImpressionNotFound = + statsReceiver.counter("ntab_impression_not_found") + private[this] val nTabOverrideOverriddenStat = + statsReceiver.counter("ntab_override_overridden") + private[this] val storeGenericNotifOverrideKey = + statsReceiver.counter("ntab_store_generic_notif_key") + private[this] val prevGenericNotifKeyNotFound = + statsReceiver.counter("ntab_prev_generic_notif_key_not_found") + + private[this] val ntabOverride = + statsReceiver.scope("ntab_override") + private[this] val ntabRequestWithOverrideId = + ntabOverride.counter("request") + private[this] val storeGenericNotifOverrideKeyWithOverrideId = + ntabOverride.counter("store_override_key") + + def send( + candidate: PushCandidate, + isNtabOnlyNotification: Boolean + ): Future[Option[CreateGenericNotificationResponse]] = { + if (candidate.target.params(FSParams.EnableOverrideIdNTabRequest)) { + ntabRequestWithOverrideId.incr() + overridePreviousEntry(candidate).flatMap { _ => + if (shouldDisableNtabOverride(candidate)) { + sendNewEntry(candidate, isNtabOnlyNotification, None) + } else { + sendNewEntry(candidate, isNtabOnlyNotification, Some(OverrideCandidate.One)) + } + } + } else { + for { + notificationOverwritten <- overrideNSlot(candidate) + _ <- deleteCachedApiTimeline(candidate, notificationOverwritten) + gnResponse <- sendNewEntry(candidate, isNtabOnlyNotification) + } yield gnResponse + } + } + + private def sendNewEntry( + candidate: PushCandidate, + isNtabOnlyNotif: Boolean, + overrideId: Option[String] = None + ): Future[Option[CreateGenericNotificationResponse]] = { + notificationServiceSender + .get( + NotificationServiceRequest( + candidate, + impressionId = candidate.impressionId, + isBadgeUpdate = isNtabOnlyNotif, + overrideId = overrideId + )).flatMap { + case Some(response) => + storeGenericNotifKey(candidate, response, overrideId).map { _ => Some(response) } + case _ => Future.None + } + } + + private def storeGenericNotifKey( + candidate: PushCandidate, + createGenericNotificationResponse: CreateGenericNotificationResponse, + overrideId: Option[String] + ): Future[Unit] = { + if (candidate.target.params(FSParams.EnableStoringNtabGenericNotifKey)) { + createGenericNotificationResponse.successKey match { + case Some(genericNotificationKey) => + val userId = genericNotificationKey.userId + if (overrideId.nonEmpty) { + storeGenericNotifOverrideKeyWithOverrideId.incr() + } + val gnOverrideKey = GenericNotificationOverrideKey( + userId = userId, + hashKey = genericNotificationKey.hashKey, + timestampMillis = genericNotificationKey.timestampMillis, + overrideId = overrideId + ) + val mhKeyVal = + ((userId, candidate.impressionId), gnOverrideKey) + storeGenericNotifOverrideKey.incr() + nTabHistoryStore.put(mhKeyVal) + case _ => Future.Unit + } + } else Future.Unit + } + + private def candidateEligibleForOverride( + targetHistory: History, + targetEntries: Seq[FrigateNotification], + ): FrigateNotification = { + val timestampToEntriesMap = + targetEntries.map { entry => + PushOverrideInfo + .getTimestampInMillisForFrigateNotification(entry, targetHistory, statsReceiver) + .getOrElse(PushConstants.DefaultLookBackForHistory.ago.inMilliseconds) -> entry + }.toMap + + PushOverrideInfo.getOldestFrigateNotification(timestampToEntriesMap) + } + + private def overrideNSlot(candidate: PushCandidate): Future[Boolean] = { + if (candidate.target.params(FSParams.EnableNslotsForOverrideOnNtab)) { + val targetHistoryFut = candidate.target.history + targetHistoryFut.flatMap { targetHistory => + val nonEligibleOverrideTypes = + Seq(RecTypes.RecommendedSpaceFanoutTypes ++ RecTypes.ScheduledSpaceReminderTypes) + + val overrideNotifs = PushOverrideInfo + .getOverrideEligiblePushNotifications( + targetHistory, + candidate.target.params(FSParams.OverrideNotificationsLookbackDurationForNTab), + statsReceiver + ).filterNot { + case notification => + nonEligibleOverrideTypes.contains(notification.commonRecommendationType) + } + + val maxNumUnreadEntries = + candidate.target.params(FSParams.OverrideNotificationsMaxCountForNTab) + if (overrideNotifs.nonEmpty && overrideNotifs.size >= maxNumUnreadEntries) { + val eligibleOverrideCandidateOpt = candidateEligibleForOverride( + targetHistory, + overrideNotifs + ) + eligibleOverrideCandidateOpt match { + case overrideCandidate if overrideCandidate.impressionId.nonEmpty => + deleteNTabEntryFromGenericNotificationStore( + candidate.target.targetId, + eligibleOverrideCandidateOpt.impressionId.head) + case _ => + ntabOverrideImpressionNotFound.incr() + Future.False + } + } else Future.False + } + } else { + Future.False + } + } + + private def shouldDisableNtabOverride(candidate: PushCandidate): Boolean = + RecTypes.isSendHandlerType(candidate.commonRecType) + + private def overridePreviousEntry(candidate: PushCandidate): Future[Boolean] = { + + if (shouldDisableNtabOverride(candidate)) { + nTabOverrideOverriddenStat.incr() + Future.False + } else { + val targetHistoryFut = candidate.target.history + targetHistoryFut.flatMap { targetHistory => + val impressionIds = PushOverrideInfo.getImpressionIdsOfPrevEligiblePushNotif( + targetHistory, + candidate.target.params(FSParams.OverrideNotificationsLookbackDurationForImpressionId), + statsReceiver) + + if (impressionIds.nonEmpty) { + deleteNTabEntryFromGenericNotificationStore(candidate.target.targetId, impressionIds.head) + } else { + ntabOverrideImpressionNotFound.incr() + Future.False // no deletes issued + } + } + } + } + + private def deleteCachedApiTimeline( + candidate: PushCandidate, + isNotificationOverridden: Boolean + ): Future[Unit] = { + if (isNotificationOverridden && candidate.target.params(FSParams.EnableDeletingNtabTimeline)) { + val deleteTimelineRequest = DeleteCurrentTimelineForUserRequest(candidate.target.targetId) + nTabDeleteTimelineRequests.incr() + nTabDeleteTimeline(deleteTimelineRequest) + } else { + Future.Unit + } + } + + private def deleteNTabEntryFromGenericNotificationStore( + targetUserId: Long, + targetImpressionId: String + ): Future[Boolean] = { + val mhKey = (targetUserId, targetImpressionId) + val genericNotificationKeyFut = nTabHistoryStore.get(mhKey) + genericNotificationKeyFut.flatMap { + case Some(genericNotifOverrideKey) => + val gnKey = GenericNotificationKey( + userId = genericNotifOverrideKey.userId, + hashKey = genericNotifOverrideKey.hashKey, + timestampMillis = genericNotifOverrideKey.timestampMillis + ) + val deleteEntryRequest = DeleteGenericNotificationRequest(gnKey) + nTabDeleteRequests.incr() + nTabDelete(deleteEntryRequest).map(_ => true) + case _ => + prevGenericNotifKeyNotFound.incr() + Future.False + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.scala new file mode 100644 index 0000000000..9690e9badb --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.scala @@ -0,0 +1,98 @@ +package com.twitter.frigate.pushservice.target + +import com.twitter.featureswitches.FSCustomMapInput +import com.twitter.featureswitches.parsing.DynMap +import com.twitter.frigate.common.store.deviceinfo.DeviceInfo +import com.twitter.frigate.pushservice.util.NsfwInfo +import com.twitter.gizmoduck.thriftscala.User + +object CustomFSFields { + private val IsReturningUser = "is_returning_user" + private val DaysSinceSignup = "days_since_signup" + private val DaysSinceLogin = "days_since_login" + private val DaysSinceReactivation = "days_since_reactivation" + private val ReactivationDate = "reactivation_date" + private val FollowGraphSize = "follow_graph_size" + private val GizmoduckUserType = "gizmoduck_user_type" + private val UserAge = "mr_user_age" + private val SensitiveOptIn = "sensitive_opt_in" + private val NsfwFollowRatio = "nsfw_follow_ratio" + private val TotalFollows = "follow_count" + private val NsfwRealGraphScore = "nsfw_real_graph_score" + private val NsfwProfileVisit = "nsfw_profile_visit" + private val TotalSearches = "total_searches" + private val NsfwSearchScore = "nsfw_search_score" + private val HasReportedNsfw = "nsfw_reported" + private val HasDislikedNsfw = "nsfw_disliked" + private val UserState = "user_state" + private val MrUserState = "mr_user_state" + private val NumDaysReceivedPushInLast30Days = + "num_days_received_push_in_last_30_days" + private val RecommendationsSetting = "recommendations_setting" + private val TopicsSetting = "topics_setting" + private val SpacesSetting = "spaces_setting" + private val NewsSetting = "news_setting" + private val LiveVideoSetting = "live_video_setting" + private val HasRecentPushableRebDevice = "has_recent_pushable_rweb_device" + private val RequestSource = "request_source" +} + +case class CustomFSFields( + isReactivatedUser: Boolean, + daysSinceSignup: Int, + numDaysReceivedPushInLast30Days: Int, + daysSinceLogin: Option[Int], + daysSinceReactivation: Option[Int], + user: Option[User], + userState: Option[String], + mrUserState: Option[String], + reactivationDate: Option[String], + requestSource: Option[String], + userAge: Option[Int], + nsfwInfo: Option[NsfwInfo], + deviceInfo: Option[DeviceInfo]) { + + import CustomFSFields._ + + private val keyValMap: Map[String, Any] = Map( + IsReturningUser -> isReactivatedUser, + DaysSinceSignup -> daysSinceSignup, + DaysSinceLogin -> daysSinceLogin, + NumDaysReceivedPushInLast30Days -> numDaysReceivedPushInLast30Days + ) ++ + daysSinceReactivation.map(DaysSinceReactivation -> _) ++ + reactivationDate.map(ReactivationDate -> _) ++ + user.flatMap(_.counts.map(counts => FollowGraphSize -> counts.following)) ++ + user.map(u => GizmoduckUserType -> u.userType.name) ++ + userState.map(UserState -> _) ++ + mrUserState.map(MrUserState -> _) ++ + requestSource.map(RequestSource -> _) ++ + userAge.map(UserAge -> _) ++ + nsfwInfo.flatMap(_.senstiveOptIn).map(SensitiveOptIn -> _) ++ + nsfwInfo + .map { nsInfo => + Map[String, Any]( + NsfwFollowRatio -> nsInfo.nsfwFollowRatio, + TotalFollows -> nsInfo.totalFollowCount, + NsfwRealGraphScore -> nsInfo.realGraphScore, + NsfwProfileVisit -> nsInfo.nsfwProfileVisits, + TotalSearches -> nsInfo.totalSearches, + NsfwSearchScore -> nsInfo.searchNsfwScore, + HasReportedNsfw -> nsInfo.hasReported, + HasDislikedNsfw -> nsInfo.hasDisliked + ) + }.getOrElse(Map.empty[String, Any]) ++ + deviceInfo + .map { deviceInfo => + Map[String, Boolean]( + RecommendationsSetting -> deviceInfo.isRecommendationsEligible, + TopicsSetting -> deviceInfo.isTopicsEligible, + SpacesSetting -> deviceInfo.isSpacesEligible, + LiveVideoSetting -> deviceInfo.isBroadcastsEligible, + NewsSetting -> deviceInfo.isNewsEligible, + HasRecentPushableRebDevice -> deviceInfo.hasRecentPushableRWebDevice + ) + }.getOrElse(Map.empty[String, Boolean]) + + val fsMap = FSCustomMapInput(DynMap(keyValMap)) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.scala new file mode 100644 index 0000000000..facbaede0c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.scala @@ -0,0 +1,182 @@ +package com.twitter.frigate.pushservice.target + +import com.twitter.abdecider.LoggingABDecider +import com.twitter.conversions.DurationOps._ +import com.twitter.decider.Decider +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.FeatureMap +import com.twitter.frigate.common.history.History +import com.twitter.frigate.common.history.HistoryStoreKeyContext +import com.twitter.frigate.common.history.MagicFanoutReasonHistory +import com.twitter.frigate.common.history.PushServiceHistoryStore +import com.twitter.frigate.common.history.RecItems +import com.twitter.frigate.common.store.deviceinfo.DeviceInfo +import com.twitter.frigate.common.util.ABDeciderWithOverride +import com.twitter.frigate.common.util.LanguageLocaleUtil +import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore +import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue +import com.twitter.frigate.dau_model.thriftscala.DauProbability +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.thriftscala.PushContext +import com.twitter.frigate.thriftscala.UserForPushTargeting +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.stp.thriftscala.STPResult +import com.twitter.interests.thriftscala.InterestId +import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue +import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails +import com.twitter.nrel.hydration.push.HydrationContext +import com.twitter.permissions_storage.thriftscala.AppPermission +import com.twitter.service.metastore.gen.thriftscala.Location +import com.twitter.service.metastore.gen.thriftscala.UserLanguages +import com.twitter.stitch.Stitch +import com.twitter.storehaus.ReadableStore +import com.twitter.strato.columns.frigate.logged_out_web_notifications.thriftscala.LOWebNotificationMetadata +import com.twitter.timelines.configapi +import com.twitter.timelines.configapi.Params +import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphFeatures +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.wtf.scalding.common.thriftscala.UserFeatures + +case class LoggedOutPushTargetUserBuilder( + historyStore: PushServiceHistoryStore, + inputDecider: Decider, + inputAbDecider: LoggingABDecider, + loggedOutPushInfoStore: ReadableStore[Long, LOWebNotificationMetadata] +)( + globalStatsReceiver: StatsReceiver) { + private val stats = globalStatsReceiver.scope("LORefreshForPushHandler") + private val noHistoryCounter = stats.counter("no_logged_out_history") + private val historyFoundCounter = stats.counter("logged_out_history_counter") + private val noLoggedOutUserCounter = stats.counter("no_logged_out_user") + private val countryCodeCounter = stats.counter("country_counter") + private val noCountryCodeCounter = stats.counter("no_country_counter") + private val noLanguageCodeCounter = stats.counter("no_language_counter") + + def buildTarget( + guestId: Long, + inputPushContext: Option[PushContext] + ): Future[Target] = { + + val historyStoreKeyContext = HistoryStoreKeyContext( + guestId, + inputPushContext.flatMap(_.useMemcacheForHistory).getOrElse(false) + ) + if (historyStore.get(historyStoreKeyContext, Some(30.days)) == Future.None) { + noHistoryCounter.incr() + } else { + historyFoundCounter.incr() + + } + if (loggedOutPushInfoStore.get(guestId) == Future.None) { + noLoggedOutUserCounter.incr() + } + Future + .join( + historyStore.get(historyStoreKeyContext, Some(30.days)), + loggedOutPushInfoStore.get(guestId) + ).map { + case (loNotifHistory, loggedOutUserPushInfo) => + new Target { + override lazy val stats: StatsReceiver = globalStatsReceiver + override val targetId: Long = guestId + override val targetGuestId = Some(guestId) + override lazy val decider: Decider = inputDecider + override lazy val loggedOutMetadata = Future.value(loggedOutUserPushInfo) + val rawLanguageFut = loggedOutMetadata.map { metadata => metadata.map(_.language) } + override val targetLanguage: Future[Option[String]] = rawLanguageFut.map { rawLang => + if (rawLang.isDefined) { + val lang = LanguageLocaleUtil.getStandardLanguageCode(rawLang.get) + if (lang.isEmpty) { + noLanguageCodeCounter.incr() + None + } else { + Option(lang) + } + } else None + } + val country = loggedOutMetadata.map(_.map(_.countryCode)) + if (country.isDefined) { + countryCodeCounter.incr() + } else { + noCountryCodeCounter.incr() + } + if (loNotifHistory == null) { + noHistoryCounter.incr() + } else { + historyFoundCounter.incr() + } + override lazy val location: Future[Option[Location]] = country.map { + case Some(code) => + Some( + Location( + city = "", + region = "", + countryCode = code, + confidence = 0.0, + lat = None, + lon = None, + metro = None, + placeIds = None, + weightedLocations = None, + createdAtMsec = None, + ip = None, + isSignupIp = None, + placeMap = None + )) + case _ => None + } + + override lazy val pushContext: Option[PushContext] = inputPushContext + override lazy val history: Future[History] = Future.value(loNotifHistory) + override lazy val magicFanoutReasonHistory30Days: Future[MagicFanoutReasonHistory] = + Future.value(null) + override lazy val globalStats: StatsReceiver = globalStatsReceiver + override lazy val pushTargeting: Future[Option[UserForPushTargeting]] = Future.None + override lazy val appPermissions: Future[Option[AppPermission]] = Future.None + override lazy val lastHTLVisitTimestamp: Future[Option[Long]] = Future.None + override lazy val pushRecItems: Future[RecItems] = Future.value(null) + + override lazy val isNewSignup: Boolean = false + override lazy val metastoreLanguages: Future[Option[UserLanguages]] = Future.None + override lazy val optOutUserInterests: Future[Option[Seq[InterestId]]] = Future.None + override lazy val mrRequestContextForFeatureStore: MrRequestContextForFeatureStore = + null + override lazy val targetUser: Future[Option[User]] = Future.None + override lazy val notificationFeedbacks: Future[Option[Seq[FeedbackPromptValue]]] = + Future.None + override lazy val promptFeedbacks: Stitch[Seq[FeedbackPromptValue]] = null + override lazy val seedsWithWeight: Future[Option[Map[Long, Double]]] = Future.None + override lazy val tweetImpressionResults: Future[Seq[Long]] = Future.Nil + override lazy val params: configapi.Params = Params.Empty + override lazy val deviceInfo: Future[Option[DeviceInfo]] = Future.None + override lazy val userFeatures: Future[Option[UserFeatures]] = Future.None + override lazy val isOpenAppExperimentUser: Future[Boolean] = Future.False + override lazy val featureMap: Future[FeatureMap] = Future.value(null) + override lazy val dauProbability: Future[Option[DauProbability]] = Future.None + override lazy val labeledPushRecsHydrated: Future[Option[UserHistoryValue]] = + Future.None + override lazy val onlineLabeledPushRecs: Future[Option[UserHistoryValue]] = Future.None + override lazy val realGraphFeatures: Future[Option[RealGraphFeatures]] = Future.None + override lazy val stpResult: Future[Option[STPResult]] = Future.None + override lazy val globalOptoutProbabilities: Seq[Future[Option[Double]]] = Seq.empty + override lazy val bucketOptoutProbability: Future[Option[Double]] = Future.None + override lazy val utcOffset: Future[Option[Duration]] = Future.None + override lazy val abDecider: ABDeciderWithOverride = + ABDeciderWithOverride(inputAbDecider, ddgOverrideOption)(globalStatsReceiver) + override lazy val resurrectionDate: Future[Option[String]] = Future.None + override lazy val isResurrectedUser: Boolean = false + override lazy val timeSinceResurrection: Option[Duration] = None + override lazy val inlineActionHistory: Future[Seq[(Long, String)]] = Future.Nil + override lazy val caretFeedbacks: Future[Option[Seq[CaretFeedbackDetails]]] = + Future.None + + override def targetHydrationContext: Future[HydrationContext] = Future.value(null) + override def isBlueVerified: Future[Option[Boolean]] = Future.None + override def isVerified: Future[Option[Boolean]] = Future.None + override def isSuperFollowCreator: Future[Option[Boolean]] = Future.None + } + } + } + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.scala new file mode 100644 index 0000000000..8378500af7 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.scala @@ -0,0 +1,694 @@ +package com.twitter.frigate.pushservice.target + +import com.twitter.abdecider.LoggingABDecider +import com.twitter.conversions.DurationOps._ +import com.twitter.decider.Decider +import com.twitter.discovery.common.configapi.ConfigParamsBuilder +import com.twitter.discovery.common.configapi.ExperimentOverride +import com.twitter.featureswitches.Recipient +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.history._ +import com.twitter.frigate.common.logger.MRLogger +import com.twitter.frigate.common.store.FeedbackRequest +import com.twitter.frigate.common.store.PushRecItemsKey +import com.twitter.frigate.common.store.deviceinfo.DeviceInfo +import com.twitter.frigate.common.store.interests.UserId +import com.twitter.frigate.common.util._ +import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore +import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue +import com.twitter.frigate.dau_model.thriftscala.DauProbability +import com.twitter.frigate.pushcap.thriftscala.PushcapInfo +import com.twitter.frigate.pushcap.thriftscala.PushcapUserHistory +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.ml.HydrationContextBuilder +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.store.LabeledPushRecsStoreKey +import com.twitter.frigate.pushservice.store.OnlineUserHistoryKey +import com.twitter.frigate.pushservice.util.NsfwInfo +import com.twitter.frigate.pushservice.util.NsfwPersonalizationUtil +import com.twitter.frigate.pushservice.util.PushAppPermissionUtil +import com.twitter.frigate.pushservice.util.PushCapUtil.getMinimumRestrictedPushcapInfo +import com.twitter.frigate.pushservice.thriftscala.PushContext +import com.twitter.frigate.pushservice.thriftscala.RequestSource +import com.twitter.frigate.thriftscala.SecondaryAccountsByUserState +import com.twitter.frigate.thriftscala.UserForPushTargeting +import com.twitter.frigate.user_states.thriftscala.MRUserHmmState +import com.twitter.frigate.user_states.thriftscala.{UserState => MrUserState} +import com.twitter.frontpage.stream.util.SnowflakeUtil +import com.twitter.geoduck.common.thriftscala.Place +import com.twitter.geoduck.service.thriftscala.LocationResponse +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.model.user_state.UserState +import com.twitter.hermit.model.user_state.UserState.UserState +import com.twitter.hermit.stp.thriftscala.STPResult +import com.twitter.ibis.thriftscala.ContentRecData +import com.twitter.interests.thriftscala.InterestId +import com.twitter.notificationservice.feedback.thriftscala.FeedbackInteraction +import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue +import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStore +import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStoreException +import com.twitter.notificationservice.model.service.DismissMenuFeedbackAction +import com.twitter.notificationservice.scribe.manhattan.GenericNotificationsFeedbackRequest +import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails +import com.twitter.nrel.heavyranker.FeatureHydrator +import com.twitter.nrel.hydration.push.HydrationContext +import com.twitter.permissions_storage.thriftscala.AppPermission +import com.twitter.rux.common.strato.thriftscala.UserTargetingProperty +import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation +import com.twitter.service.metastore.gen.thriftscala.Location +import com.twitter.service.metastore.gen.thriftscala.UserLanguages +import com.twitter.stitch.Stitch +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.configapi +import com.twitter.timelines.real_graph.thriftscala.{RealGraphFeatures => RealGraphFeaturesUnion} +import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphFeatures +import com.twitter.ubs.thriftscala.SellerApplicationState +import com.twitter.ubs.thriftscala.SellerTrack +import com.twitter.user_session_store.thriftscala.UserSession +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.util.Time +import com.twitter.wtf.scalding.common.thriftscala.UserFeatures + +case class PushTargetUserBuilder( + historyStore: PushServiceHistoryStore, + emailHistoryStore: PushServiceHistoryStore, + labeledPushRecsStore: ReadableStore[LabeledPushRecsStoreKey, UserHistoryValue], + onlineUserHistoryStore: ReadableStore[OnlineUserHistoryKey, UserHistoryValue], + pushRecItemsStore: ReadableStore[PushRecItemsKey, RecItems], + userStore: ReadableStore[Long, User], + pushInfoStore: ReadableStore[Long, UserForPushTargeting], + userCountryStore: ReadableStore[Long, Location], + userUtcOffsetStore: ReadableStore[Long, Duration], + dauProbabilityStore: ReadableStore[Long, DauProbability], + nsfwConsumerStore: ReadableStore[Long, NSFWUserSegmentation], + userFeatureStore: ReadableStore[Long, UserFeatures], + userTargetingPropertyStore: ReadableStore[Long, UserTargetingProperty], + mrUserStateStore: ReadableStore[Long, MRUserHmmState], + tweetImpressionStore: ReadableStore[Long, Seq[Long]], + ntabCaretFeedbackStore: ReadableStore[GenericNotificationsFeedbackRequest, Seq[ + CaretFeedbackDetails + ]], + genericFeedbackStore: ReadableStore[FeedbackRequest, Seq[FeedbackPromptValue]], + genericNotificationFeedbackStore: GenericFeedbackStore, + timelinesUserSessionStore: ReadableStore[Long, UserSession], + cachedTweetyPieStore: ReadableStore[Long, TweetyPieResult], + strongTiesStore: ReadableStore[Long, STPResult], + userHTLLastVisitStore: ReadableStore[Long, Seq[Long]], + userLanguagesStore: ReadableStore[Long, UserLanguages], + inputDecider: Decider, + inputAbDecider: LoggingABDecider, + realGraphScoresTop500InStore: ReadableStore[Long, Map[Long, Double]], + recentFollowsStore: ReadableStore[Long, Seq[Long]], + resurrectedUserStore: ReadableStore[Long, String], + configParamsBuilder: ConfigParamsBuilder, + optOutUserInterestsStore: ReadableStore[UserId, Seq[InterestId]], + deviceInfoStore: ReadableStore[Long, DeviceInfo], + pushcapDynamicPredictionStore: ReadableStore[Long, PushcapUserHistory], + appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission], + optoutModelScorer: PushMLModelScorer, + inlineActionHistoryStore: ReadableStore[Long, Seq[(Long, String)]], + featureHydrator: FeatureHydrator, + openAppUserStore: ReadableStore[Long, Boolean], + openedPushByHourAggregatedStore: ReadableStore[Long, Map[Int, Int]], + geoduckStoreV2: ReadableStore[Long, LocationResponse], + superFollowEligibilityUserStore: ReadableStore[Long, Boolean], + superFollowApplicationStatusStore: ReadableStore[(Long, SellerTrack), SellerApplicationState] +)( + globalStatsReceiver: StatsReceiver) { + + implicit val statsReceiver: StatsReceiver = globalStatsReceiver + + private val log = MRLogger("PushTargetUserBuilder") + private val recentFollowscounter = statsReceiver.counter("query_recent_follows") + private val isModelTrainingDataCounter = + statsReceiver.scope("TargetUserBuilder").counter("is_model_training") + private val feedbackStoreGenerationErr = statsReceiver.counter("feedback_store_generation_error") + private val newSignUpUserStats = statsReceiver.counter("new_signup_user") + private val pushcapSelectionStat = statsReceiver.scope("pushcap_modeling") + private val dormantUserCount = statsReceiver.counter("dormant_user_counter") + private val optoutModelStat = statsReceiver.scope("optout_modeling") + private val placeFoundStat = statsReceiver.scope("geoduck_v2").stat("places_found") + private val placesNotFound = statsReceiver.scope("geoduck_v2").counter("places_not_found") + // Email history store stats + private val emailHistoryStats = statsReceiver.scope("email_tweet_history") + private val emptyEmailHistoryCounter = emailHistoryStats.counter("empty") + private val nonEmptyEmailHistoryCounter = emailHistoryStats.counter("non_empty") + + private val MagicRecsCategory = "MagicRecs" + private val MomentsViaMagicRecsCategory = "MomentsViaMagicRecs" + private val MomentsCategory = "Moments" + + def buildTarget( + userId: Long, + inputPushContext: Option[PushContext], + forcedFeatureValues: Option[Map[String, configapi.FeatureValue]] = None + ): Future[Target] = { + val historyStoreKeyContext = HistoryStoreKeyContext( + userId, + inputPushContext.flatMap(_.useMemcacheForHistory).getOrElse(false) + ) + Future + .join( + userStore.get(userId), + deviceInfoStore.get(userId), + pushInfoStore.get(userId), + historyStore.get(historyStoreKeyContext, Some(30.days)), + emailHistoryStore.get( + HistoryStoreKeyContext(userId, useStoreB = false), + Some(7.days) // we only keep 7 days of email tweet history + ) + ).flatMap { + case (userOpt, deviceInfoOpt, userForPushTargetingInfoOpt, notifHistory, emailHistory) => + getCustomFSFields( + userId, + userOpt, + deviceInfoOpt, + userForPushTargetingInfoOpt, + notifHistory, + inputPushContext.flatMap(_.requestSource)).map { customFSField => + new Target { + + override lazy val stats: StatsReceiver = statsReceiver + + override val targetId: Long = userId + + override val targetUser: Future[Option[User]] = Future.value(userOpt) + + override val isEmailUser: Boolean = + inputPushContext.flatMap(_.requestSource) match { + case Some(source) if source == RequestSource.Email => true + case _ => false + } + + override val pushContext = inputPushContext + + override def globalStats: StatsReceiver = globalStatsReceiver + + override lazy val abDecider: ABDeciderWithOverride = + ABDeciderWithOverride(inputAbDecider, ddgOverrideOption) + + override lazy val pushRecItems: Future[RecItems] = + pushRecItemsStore + .get(PushRecItemsKey(historyStoreKeyContext, history)) + .map(_.getOrElse(RecItems.empty)) + + // List of past tweet candidates sent in the past through email with timestamp + override lazy val emailRecItems: Future[Seq[(Time, Long)]] = { + Future.value { + emailHistory.sortedEmailHistory.flatMap { + case (timeStamp, notification) => + notification.contentRecsNotification + .map { notification => + notification.recommendations.contentRecCollections.flatMap { + contentRecs => + contentRecs.contentRecModules.flatMap { contentRecModule => + contentRecModule.recData match { + case ContentRecData.TweetRec(tweetRec) => + nonEmptyEmailHistoryCounter.incr() + Seq(tweetRec.tweetId) + case _ => + emptyEmailHistoryCounter.incr() + Nil + } + } + } + }.getOrElse { + emptyEmailHistoryCounter.incr() + Nil + }.map(timeStamp -> _) + } + } + } + + override lazy val history: Future[History] = Future.value(notifHistory) + + override lazy val pushTargeting: Future[Option[UserForPushTargeting]] = + Future.value(userForPushTargetingInfoOpt) + + override lazy val decider: Decider = inputDecider + + override lazy val location: Future[Option[Location]] = + userCountryStore.get(userId) + + override lazy val deviceInfo: Future[Option[DeviceInfo]] = + Future.value(deviceInfoOpt) + + override lazy val targetLanguage: Future[Option[String]] = targetUser map { userOpt => + userOpt.flatMap(_.account.map(_.language)) + } + + override lazy val targetAgeInYears: Future[Option[Int]] = + Future.value(customFSField.userAge) + + override lazy val metastoreLanguages: Future[Option[UserLanguages]] = + userLanguagesStore.get(targetId) + + override lazy val utcOffset: Future[Option[Duration]] = + userUtcOffsetStore.get(targetId) + + override lazy val userFeatures: Future[Option[UserFeatures]] = + userFeatureStore.get(targetId) + + override lazy val targetUserState: Future[Option[UserState]] = + Future.value( + customFSField.userState + .flatMap(userState => UserState.valueOf(userState))) + + override lazy val targetMrUserState: Future[Option[MrUserState]] = + Future.value( + customFSField.mrUserState + .flatMap(mrUserState => MrUserState.valueOf(mrUserState))) + + override lazy val accountStateWithDeviceInfo: Future[ + Option[SecondaryAccountsByUserState] + ] = Future.None + + override lazy val dauProbability: Future[Option[DauProbability]] = { + dauProbabilityStore.get(targetId) + } + + override lazy val labeledPushRecsHydrated: Future[Option[UserHistoryValue]] = + labeledPushRecsStore.get(LabeledPushRecsStoreKey(this, historyStoreKeyContext)) + + override lazy val onlineLabeledPushRecs: Future[Option[UserHistoryValue]] = + labeledPushRecsHydrated.flatMap { labeledPushRecs => + history.flatMap { history => + onlineUserHistoryStore.get( + OnlineUserHistoryKey(targetId, labeledPushRecs, Some(history)) + ) + } + } + + override lazy val tweetImpressionResults: Future[Seq[Long]] = + tweetImpressionStore.get(targetId).map { + case Some(impressionList) => + impressionList + case _ => Nil + } + + override lazy val realGraphFeatures: Future[Option[RealGraphFeatures]] = + timelinesUserSessionStore.get(targetId).map { userSessionOpt => + userSessionOpt.flatMap { userSession => + userSession.realGraphFeatures.collect { + case RealGraphFeaturesUnion.V1(rGFeatures) => + rGFeatures + } + } + } + + override lazy val stpResult: Future[Option[STPResult]] = + strongTiesStore.get(targetId) + + override lazy val lastHTLVisitTimestamp: Future[Option[Long]] = + userHTLLastVisitStore.get(targetId).map { + case Some(lastVisitTimestamps) if lastVisitTimestamps.nonEmpty => + Some(lastVisitTimestamps.max) + case _ => None + } + + override lazy val caretFeedbacks: Future[Option[Seq[CaretFeedbackDetails]]] = { + val scribeHistoryLookbackPeriod = 365.days + val now = Time.now + val request = GenericNotificationsFeedbackRequest( + userId = targetId, + eventStartTimestamp = now - scribeHistoryLookbackPeriod, + eventEndTimestamp = now, + filterCategory = + Some(Set(MagicRecsCategory, MomentsViaMagicRecsCategory, MomentsCategory)), + filterFeedbackActionText = + Some(Set(DismissMenuFeedbackAction.FeedbackActionTextSeeLessOften)) + ) + ntabCaretFeedbackStore.get(request) + } + + override lazy val notificationFeedbacks: Future[ + Option[Seq[FeedbackPromptValue]] + ] = { + val scribeHistoryLookbackPeriod = 30.days + val now = Time.now + val request = FeedbackRequest( + userId = targetId, + oldestTimestamp = scribeHistoryLookbackPeriod.ago, + newestTimestamp = Time.now, + feedbackInteraction = FeedbackInteraction.Feedback + ) + genericFeedbackStore.get(request) + } + + // DEPRECATED: Use notificationFeedbacks instead. + // This method will increase latency dramatically. + override lazy val promptFeedbacks: Stitch[Seq[FeedbackPromptValue]] = { + val scribeHistoryLookbackPeriod = 7.days + + genericNotificationFeedbackStore + .getAll( + userId = targetId, + oldestTimestamp = scribeHistoryLookbackPeriod.ago, + newestTimestamp = Time.now, + feedbackInteraction = FeedbackInteraction.Feedback + ).handle { + case _: GenericFeedbackStoreException => { + feedbackStoreGenerationErr.incr() + Seq.empty[FeedbackPromptValue] + } + } + } + + override lazy val optOutUserInterests: Future[Option[Seq[InterestId]]] = { + optOutUserInterestsStore.get(targetId) + } + + private val experimentOverride = ddgOverrideOption.map { + case DDGOverride(Some(exp), Some(bucket)) => + Set(ExperimentOverride(exp, bucket)) + case _ => Set.empty[ExperimentOverride] + } + + override val signupCountryCode = + Future.value(userOpt.flatMap(_.safety.flatMap(_.signupCountryCode))) + + override lazy val params: configapi.Params = { + val fsRecipient = Recipient( + userId = Some(targetId), + userRoles = userOpt.flatMap(_.roles.map(_.roles.toSet)), + clientApplicationId = deviceInfoOpt.flatMap(_.guessedPrimaryClientAppId), + userAgent = deviceInfoOpt.flatMap(_.guessedPrimaryDeviceUserAgent), + countryCode = + userOpt.flatMap(_.account.flatMap(_.countryCode.map(_.toUpperCase))), + customFields = Some(customFSField.fsMap), + signupCountryCode = + userOpt.flatMap(_.safety.flatMap(_.signupCountryCode.map(_.toUpperCase))), + languageCode = deviceInfoOpt.flatMap { + _.deviceLanguages.flatMap(IbisAppPushDeviceSettingsUtil.inferredDeviceLanguage) + } + ) + + configParamsBuilder.build( + userId = Some(targetId), + experimentOverrides = experimentOverride, + featureRecipient = Some(fsRecipient), + forcedFeatureValues = forcedFeatureValues.getOrElse(Map.empty), + ) + } + + override lazy val mrRequestContextForFeatureStore = + MrRequestContextForFeatureStore(targetId, params, isModelTrainingData) + + override lazy val dynamicPushcap: Future[Option[PushcapInfo]] = { + // Get the pushcap from the pushcap model prediction store + if (params(PushParams.EnableModelBasedPushcapAssignments)) { + val originalPushcapInfoFut = + PushCapUtil.getPushcapFromUserHistory( + userId, + pushcapDynamicPredictionStore, + params(FeatureSwitchParams.PushcapModelType), + params(FeatureSwitchParams.PushcapModelPredictionVersion), + pushcapSelectionStat + ) + // Modify the push cap info if there is a restricted min value for predicted push caps. + val restrictedPushcap = params(PushFeatureSwitchParams.RestrictedMinModelPushcap) + originalPushcapInfoFut.map { + case Some(originalPushcapInfo) => + Some( + getMinimumRestrictedPushcapInfo( + restrictedPushcap, + originalPushcapInfo, + pushcapSelectionStat)) + case _ => None + } + } else Future.value(None) + } + + override lazy val targetHydrationContext: Future[HydrationContext] = + HydrationContextBuilder.build(this) + + override lazy val featureMap: Future[FeatureMap] = + targetHydrationContext.flatMap { hydrationContext => + featureHydrator.hydrateTarget( + hydrationContext, + this.params, + this.mrRequestContextForFeatureStore) + } + + override lazy val globalOptoutProbabilities: Seq[Future[Option[Double]]] = { + params(PushFeatureSwitchParams.GlobalOptoutModelParam).map { model_id => + optoutModelScorer + .singlePredictionForTargetLevel(model_id, targetId, featureMap) + } + } + + override lazy val bucketOptoutProbability: Future[Option[Double]] = { + Future + .collect(globalOptoutProbabilities).map { + _.zip(params(PushFeatureSwitchParams.GlobalOptoutThresholdParam)) + .exists { + case (Some(score), threshold) => score >= threshold + case _ => false + } + }.flatMap { + case true => + optoutModelScorer.singlePredictionForTargetLevel( + params(PushFeatureSwitchParams.BucketOptoutModelParam), + targetId, + featureMap) + case _ => Future.None + } + } + + override lazy val optoutAdjustedPushcap: Future[Option[Short]] = { + if (params(PushFeatureSwitchParams.EnableOptoutAdjustedPushcap)) { + bucketOptoutProbability.map { + case Some(score) => + val idx = params(PushFeatureSwitchParams.BucketOptoutSlotThresholdParam) + .indexWhere(score <= _) + if (idx >= 0) { + val pushcap = + params(PushFeatureSwitchParams.BucketOptoutSlotPushcapParam)(idx).toShort + optoutModelStat.scope("adjusted_pushcap").counter(f"$pushcap").incr() + if (pushcap >= 0) Some(pushcap) + else None + } else None + case _ => None + } + } else Future.None + } + + override lazy val seedsWithWeight: Future[Option[Map[Long, Double]]] = { + Future + .join( + realGraphScoresTop500InStore.get(userId), + targetUserState, + targetUser + ) + .flatMap { + case (seedSetOpt, userState, gizmoduckUser) => + val seedSet = seedSetOpt.getOrElse(Map.empty[Long, Double]) + + //If new sign_up or New user, combine recent_follows with real graph seedset + val isNewUserEnabled = { + val isNewerThan7days = customFSField.daysSinceSignup <= 7 + val isNewUserState = userState.contains(UserState.New) + isNewUserState || isNewSignup || isNewerThan7days + } + + val nonSeedSetFollowsFut = gizmoduckUser match { + case Some(user) if isNewUserEnabled => + recentFollowscounter.incr() + recentFollowsStore.get(user.id) + + case Some(user) if this.isModelTrainingData => + recentFollowscounter.incr() + isModelTrainingDataCounter.incr() + recentFollowsStore.get(user.id) + + case _ => Future.None + } + nonSeedSetFollowsFut.map { nonSeedSetFollows => + Some( + SeedsetUtil.combineRecentFollowsWithWeightedSeedset( + seedSet, + nonSeedSetFollows.getOrElse(Nil) + ) + ) + } + } + } + + override def magicFanoutReasonHistory30Days: Future[MagicFanoutReasonHistory] = + history.map(history => MagicFanoutReasonHistory(history)) + + override val isNewSignup: Boolean = + pushContext.flatMap(_.isFromNewUserLoopProcessor).getOrElse(false) + + override lazy val resurrectionDate: Future[Option[String]] = + Future.value(customFSField.reactivationDate) + + override lazy val isResurrectedUser: Boolean = + customFSField.daysSinceReactivation.isDefined + + override lazy val timeSinceResurrection: Option[Duration] = + customFSField.daysSinceReactivation.map(Duration.fromDays) + + override lazy val appPermissions: Future[Option[AppPermission]] = + PushAppPermissionUtil.getAppPermission( + userId, + PushAppPermissionUtil.AddressBookPermissionKey, + deviceInfo, + appPermissionStore) + + override lazy val inlineActionHistory: Future[Seq[(Long, String)]] = { + inlineActionHistoryStore + .get(userId).map { + case Some(sortedInlineActionHistory) => sortedInlineActionHistory + case _ => Seq.empty + } + } + + lazy val isOpenAppExperimentUser: Future[Boolean] = + openAppUserStore.get(userId).map(_.contains(true)) + + override lazy val openedPushByHourAggregated: Future[Option[Map[Int, Int]]] = + openedPushByHourAggregatedStore.get(userId) + + override lazy val places: Future[Seq[Place]] = { + geoduckStoreV2 + .get(targetId) + .map(_.flatMap(_.places)) + .map { + case Some(placeSeq) if placeSeq.nonEmpty => + placeFoundStat.add(placeSeq.size) + placeSeq + case _ => + placesNotFound.incr() + Seq.empty + } + } + + override val isBlueVerified: Future[Option[Boolean]] = + Future.value(userOpt.flatMap(_.safety.flatMap(_.isBlueVerified))) + + override val isVerified: Future[Option[Boolean]] = + Future.value(userOpt.flatMap(_.safety.map(_.verified))) + + override lazy val isSuperFollowCreator: Future[Option[Boolean]] = + superFollowEligibilityUserStore.get(targetId) + } + } + } + } + + /** + * Provide general way to add needed FS for target user, and package them in CustomFSFields. + * Custom Fields is a powerful feature that allows Feature Switch library users to define and + * match against any arbitrary fields. + **/ + private def getCustomFSFields( + userId: Long, + userOpt: Option[User], + deviceInfo: Option[DeviceInfo], + userForPushTargetingInfo: Option[UserForPushTargeting], + notifHistory: History, + requestSource: Option[RequestSource] + ): Future[CustomFSFields] = { + val reactivationDateFutOpt: Future[Option[String]] = resurrectedUserStore.get(userId) + val reactivationTimeFutOpt: Future[Option[Time]] = + reactivationDateFutOpt.map(_.map(dateStr => DateUtil.dateStrToTime(dateStr))) + + val isReactivatedUserFut: Future[Boolean] = reactivationTimeFutOpt.map { timeOpt => + timeOpt + .exists { time => Time.now - time < 30.days } + } + + val daysSinceReactivationFut: Future[Option[Int]] = + reactivationTimeFutOpt.map(_.map(time => Time.now.since(time).inDays)) + + val daysSinceSignup: Int = (Time.now - SnowflakeUtil.timeFromId(userId)).inDays + if (daysSinceSignup < 14) newSignUpUserStats.incr() + + val targetAgeInYears = userOpt.flatMap(_.extendedProfile.flatMap(_.ageInYears)) + + val lastLoginFut: Future[Option[Long]] = + userHTLLastVisitStore.get(userId).map { + case Some(lastHTLVisitTimes) => + val latestHTLVisitTime = lastHTLVisitTimes.max + userForPushTargetingInfo.flatMap( + _.lastActiveOnAppTimestamp + .map(_.max(latestHTLVisitTime)).orElse(Some(latestHTLVisitTime))) + case None => + userForPushTargetingInfo.flatMap(_.lastActiveOnAppTimestamp) + } + + val daysSinceLoginFut = lastLoginFut.map { + _.map { lastLoginTimestamp => + val timeSinceLogin = Time.now - Time.fromMilliseconds(lastLoginTimestamp) + if (timeSinceLogin.inDays > 21) { + dormantUserCount.incr() + } + timeSinceLogin.inDays + } + } + + /* Could add more custom FS here */ + val userNSFWInfoFut: Future[Option[NsfwInfo]] = + nsfwConsumerStore + .get(userId).map(_.map(nsfwUserSegmentation => NsfwInfo(nsfwUserSegmentation))) + + val userStateFut: Future[Option[String]] = userFeatureStore.get(userId).map { userFeaturesOpt => + userFeaturesOpt.flatMap { uFeats => + uFeats.userState.map(uState => uState.name) + } + } + + val mrUserStateFut: Future[Option[String]] = + mrUserStateStore.get(userId).map { mrUserStateOpt => + mrUserStateOpt.flatMap { mrUserState => + mrUserState.userState.map(_.name) + } + } + + Future + .join( + reactivationDateFutOpt, + isReactivatedUserFut, + userStateFut, + mrUserStateFut, + daysSinceLoginFut, + daysSinceReactivationFut, + userNSFWInfoFut + ).map { + case ( + reactivationDate, + isReactivatedUser, + userState, + mrUserState, + daysSinceLogin, + daysSinceReactivation, + userNSFWInfo) => + val numDaysReceivedPushInLast30Days: Int = + notifHistory.history.keys.map(_.inDays).toSet.size + + NsfwPersonalizationUtil.computeNsfwUserStats(userNSFWInfo) + + CustomFSFields( + isReactivatedUser, + daysSinceSignup, + numDaysReceivedPushInLast30Days, + daysSinceLogin, + daysSinceReactivation, + userOpt, + userState, + mrUserState, + reactivationDate, + requestSource.map(_.name), + targetAgeInYears, + userNSFWInfo, + deviceInfo + ) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.scala new file mode 100644 index 0000000000..745e061fba --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.scala @@ -0,0 +1,37 @@ +package com.twitter.frigate.pushservice.target + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.predicate.TargetPromptFeedbackFatiguePredicate +import com.twitter.frigate.common.predicate.TargetUserPredicates +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.predicate.TargetNtabCaretClickFatiguePredicate +import com.twitter.frigate.pushservice.predicate.TargetPredicates +import com.twitter.hermit.predicate.NamedPredicate + +class RFPHTargetPredicateGenerator(implicit statsReceiver: StatsReceiver) { + val predicates: List[NamedPredicate[Target]] = List( + TargetPredicates.magicRecsMinDurationSinceSent(), + TargetPredicates.targetHTLVisitPredicate(), + TargetPredicates.inlineActionFatiguePredicate(), + TargetPredicates.targetFatiguePredicate(), + TargetUserPredicates.secondaryDormantAccountPredicate(), + TargetPredicates.targetValidMobileSDKPredicate, + TargetPredicates.targetPushBitEnabledPredicate, + TargetUserPredicates.targetUserExists(), + TargetPredicates.paramPredicate(PushFeatureSwitchParams.EnablePushRecommendationsParam), + TargetPromptFeedbackFatiguePredicate.responseNoPredicate( + PushParams.EnablePromptFeedbackFatigueResponseNoPredicate, + PushConstants.AcceptableTimeSinceLastNegativeResponse), + TargetPredicates.teamExceptedPredicate(TargetNtabCaretClickFatiguePredicate.apply()), + TargetPredicates.optoutProbPredicate(), + TargetPredicates.webNotifsHoldback() + ) +} + +object RFPHTargetPredicates { + def apply(implicit statsReceiver: StatsReceiver): List[NamedPredicate[Target]] = + new RFPHTargetPredicateGenerator().predicates +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.scala new file mode 100644 index 0000000000..c84af12869 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.scala @@ -0,0 +1,10 @@ +package com.twitter.frigate.pushservice.target + +import com.twitter.permissions_storage.thriftscala.AppPermission +import com.twitter.util.Future + +trait TargetAppPermissions { + + def appPermissions: Future[Option[AppPermission]] + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.scala new file mode 100644 index 0000000000..2a9c26e8bd --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.scala @@ -0,0 +1,121 @@ +package com.twitter.frigate.pushservice.target + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.FeatureMap +import com.twitter.frigate.common.base.TargetUser +import com.twitter.frigate.common.candidate.TargetABDecider +import com.twitter.frigate.common.candidate.TargetDecider +import com.twitter.frigate.common.candidate.UserDetails +import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue +import com.twitter.frigate.dau_model.thriftscala.DauProbability +import com.twitter.frigate.scribe.thriftscala.SkipModelInfo +import com.twitter.hermit.stp.thriftscala.STPResult +import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphFeatures +import com.twitter.util.Future +import com.twitter.util.Time +import com.twitter.frigate.pushservice.params.DeciderKey +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.params.WeightedOpenOrNtabClickModel +import com.twitter.frigate.pushservice.util.PushDeviceUtil +import com.twitter.nrel.hydration.push.HydrationContext +import com.twitter.timelines.configapi.FSParam + +trait TargetScoringDetails { + tuc: TargetUser with TargetDecider with TargetABDecider with UserDetails => + + def stats: StatsReceiver + + /* + * We have 3 types of model training data: + * 1, skip ranker and model predicates + * controlled by decider frigate_notifier_quality_model_training_data + * the data distribution is same to the distribution in ranking + * 2, skip model predicates only + * controlled by decider skip_ml_model_predicate + * the data distribution is same to the distribution in filtering + * 3, no skip, only scribe features + * controlled by decider scribe_model_features + * the data distribution is same to production traffic + * The "miscellaneous" is used to store all misc information for selecting the data offline (e.g., ddg-bucket information) + * */ + lazy val skipModelInfo: Option[SkipModelInfo] = { + val trainingDataDeciderKey = DeciderKey.trainingDataDeciderKey.toString + val skipMlModelPredicateDeciderKey = DeciderKey.skipMlModelPredicateDeciderKey.toString + val scribeModelFeaturesDeciderKey = DeciderKey.scribeModelFeaturesDeciderKey.toString + val miscellaneous = None + + if (isDeciderEnabled(trainingDataDeciderKey, stats, useRandomRecipient = true)) { + Some( + SkipModelInfo( + skipPushOpenPredicate = Some(true), + skipPushRanker = Some(true), + miscellaneous = miscellaneous)) + } else if (isDeciderEnabled(skipMlModelPredicateDeciderKey, stats, useRandomRecipient = true)) { + Some( + SkipModelInfo( + skipPushOpenPredicate = Some(true), + skipPushRanker = Some(false), + miscellaneous = miscellaneous)) + } else if (isDeciderEnabled(scribeModelFeaturesDeciderKey, stats, useRandomRecipient = true)) { + Some(SkipModelInfo(noSkipButScribeFeatures = Some(true), miscellaneous = miscellaneous)) + } else { + Some(SkipModelInfo(miscellaneous = miscellaneous)) + } + } + + lazy val scribeFeatureForRequestScribe = + isDeciderEnabled( + DeciderKey.scribeModelFeaturesForRequestScribe.toString, + stats, + useRandomRecipient = true) + + lazy val rankingModelParam: Future[FSParam[WeightedOpenOrNtabClickModel.ModelNameType]] = + tuc.deviceInfo.map { deviceInfoOpt => + if (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) && + tuc.params(PushParams.AndroidOnlyRankingExperimentParam)) { + PushFeatureSwitchParams.WeightedOpenOrNtabClickRankingModelForAndroidParam + } else { + PushFeatureSwitchParams.WeightedOpenOrNtabClickRankingModelParam + } + } + + lazy val filteringModelParam: FSParam[WeightedOpenOrNtabClickModel.ModelNameType] = + PushFeatureSwitchParams.WeightedOpenOrNtabClickFilteringModelParam + + def skipMlRanker: Boolean = skipModelInfo.exists(_.skipPushRanker.contains(true)) + + def skipModelPredicate: Boolean = skipModelInfo.exists(_.skipPushOpenPredicate.contains(true)) + + def noSkipButScribeFeatures: Boolean = + skipModelInfo.exists(_.noSkipButScribeFeatures.contains(true)) + + def isModelTrainingData: Boolean = skipMlRanker || skipModelPredicate || noSkipButScribeFeatures + + def scribeFeatureWithoutHydratingNewFeatures: Boolean = + isDeciderEnabled( + DeciderKey.scribeModelFeaturesWithoutHydratingNewFeaturesDeciderKey.toString, + stats, + useRandomRecipient = true + ) + + def targetHydrationContext: Future[HydrationContext] + + def featureMap: Future[FeatureMap] + + def dauProbability: Future[Option[DauProbability]] + + def labeledPushRecsHydrated: Future[Option[UserHistoryValue]] + + def onlineLabeledPushRecs: Future[Option[UserHistoryValue]] + + def realGraphFeatures: Future[Option[RealGraphFeatures]] + + def stpResult: Future[Option[STPResult]] + + def globalOptoutProbabilities: Seq[Future[Option[Double]]] + + def bucketOptoutProbability: Future[Option[Double]] + + val sendTime: Long = Time.now.inMillis +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.scala new file mode 100644 index 0000000000..dad918023f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.scala @@ -0,0 +1,15 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.FutureOps +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +object AdaptorUtils { + def getTweetyPieResults( + tweetIds: Set[Long], + tweetyPieStore: ReadableStore[Long, TweetyPieResult], + ): Future[Map[Long, Option[TweetyPieResult]]] = + FutureOps + .mapCollect(tweetyPieStore.multiGet(tweetIds)) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.scala new file mode 100644 index 0000000000..1d3ff461e3 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.scala @@ -0,0 +1,104 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.CandidateResult +import com.twitter.frigate.common.base.Invalid +import com.twitter.frigate.common.base.OK +import com.twitter.frigate.common.base.Result +import com.twitter.frigate.common.base.TweetAuthor +import com.twitter.frigate.common.base.TweetCandidate +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.ListOfAdhocIdsForStatsTracking + +class AdhocStatsUtil(stats: StatsReceiver) { + + private def getAdhocIds(candidate: PushCandidate): Set[Long] = + candidate.target.params(ListOfAdhocIdsForStatsTracking) + + private def isAdhocTweetCandidate(candidate: PushCandidate): Boolean = { + candidate match { + case tweetCandidate: RawCandidate with TweetCandidate with TweetAuthor => + tweetCandidate.authorId.exists(id => getAdhocIds(candidate).contains(id)) + case _ => false + } + } + + def getCandidateSourceStats(hydratedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { + hydratedCandidates.foreach { hydratedCandidate => + if (isAdhocTweetCandidate(hydratedCandidate.candidate)) { + stats.scope("candidate_source").counter(hydratedCandidate.source).incr() + } + } + } + + def getPreRankingFilterStats( + preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]] + ): Unit = { + preRankingFilteredCandidates.foreach { filteredCandidate => + if (isAdhocTweetCandidate(filteredCandidate.candidate)) { + filteredCandidate.result match { + case Invalid(reason) => + stats.scope("preranking_filter").counter(reason.getOrElse("unknown_reason")).incr() + case _ => + } + } + } + } + + def getLightRankingStats(lightRankedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { + lightRankedCandidates.foreach { lightRankedCandidate => + if (isAdhocTweetCandidate(lightRankedCandidate.candidate)) { + stats.scope("light_ranker").counter("passed_light_ranking").incr() + } + } + } + + def getRankingStats(rankedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { + rankedCandidates.zipWithIndex.foreach { + case (rankedCandidate, index) => + val rankerStats = stats.scope("heavy_ranker") + if (isAdhocTweetCandidate(rankedCandidate.candidate)) { + rankerStats.counter("ranked_candidates").incr() + rankerStats.stat("rank").add(index.toFloat) + rankedCandidate.candidate.modelScores.map { modelScores => + modelScores.foreach { + case (modelName, score) => + // mutiply score by 1000 to not lose precision while converting to Float + val precisionScore = (score * 100000).toFloat + rankerStats.stat(modelName).add(precisionScore) + } + } + } + } + } + def getReRankingStats(rankedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { + rankedCandidates.zipWithIndex.foreach { + case (rankedCandidate, index) => + val rankerStats = stats.scope("re_ranking") + if (isAdhocTweetCandidate(rankedCandidate.candidate)) { + rankerStats.counter("re_ranked_candidates").incr() + rankerStats.stat("re_rank").add(index.toFloat) + } + } + } + + def getTakeCandidateResultStats( + allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]] + ): Unit = { + val takeStats = stats.scope("take_step") + allTakeCandidateResults.foreach { candidateResult => + if (isAdhocTweetCandidate(candidateResult.candidate)) { + candidateResult.result match { + case OK => + takeStats.counter("sent").incr() + case Invalid(reason) => + takeStats.counter(reason.getOrElse("unknown_reason")).incr() + case _ => + takeStats.counter("unknown_filter").incr() + } + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.scala new file mode 100644 index 0000000000..6fa0bc2882 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.scala @@ -0,0 +1,119 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.thriftscala.FrigateNotification +import com.twitter.frigate.thriftscala.NotificationDisplayLocation + +object Candidate2FrigateNotification { + + def getFrigateNotification( + candidate: PushCandidate + )( + implicit statsReceiver: StatsReceiver + ): FrigateNotification = { + candidate match { + + case topicTweetCandidate: PushCandidate with BaseTopicTweetCandidate => + PushAdaptorUtil.getFrigateNotificationForTweet( + crt = topicTweetCandidate.commonRecType, + tweetId = topicTweetCandidate.tweetId, + scActions = Nil, + authorIdOpt = topicTweetCandidate.authorId, + pushCopyId = topicTweetCandidate.pushCopyId, + ntabCopyId = topicTweetCandidate.ntabCopyId, + simclusterId = None, + semanticCoreEntityIds = topicTweetCandidate.semanticCoreEntityId.map(List(_)), + candidateContent = topicTweetCandidate.content, + trendId = None + ) + + case trendTweetCandidate: PushCandidate with TrendTweetCandidate => + PushAdaptorUtil.getFrigateNotificationForTweet( + trendTweetCandidate.commonRecType, + trendTweetCandidate.tweetId, + Nil, + trendTweetCandidate.authorId, + trendTweetCandidate.pushCopyId, + trendTweetCandidate.ntabCopyId, + None, + None, + trendTweetCandidate.content, + Some(trendTweetCandidate.trendId) + ) + + case tripTweetCandidate: PushCandidate with OutOfNetworkTweetCandidate with TripCandidate => + PushAdaptorUtil.getFrigateNotificationForTweet( + crt = tripTweetCandidate.commonRecType, + tweetId = tripTweetCandidate.tweetId, + scActions = Nil, + authorIdOpt = tripTweetCandidate.authorId, + pushCopyId = tripTweetCandidate.pushCopyId, + ntabCopyId = tripTweetCandidate.ntabCopyId, + simclusterId = None, + semanticCoreEntityIds = None, + candidateContent = tripTweetCandidate.content, + trendId = None, + tweetTripDomain = tripTweetCandidate.tripDomain + ) + + case outOfNetworkTweetCandidate: PushCandidate with OutOfNetworkTweetCandidate => + PushAdaptorUtil.getFrigateNotificationForTweet( + crt = outOfNetworkTweetCandidate.commonRecType, + tweetId = outOfNetworkTweetCandidate.tweetId, + scActions = Nil, + authorIdOpt = outOfNetworkTweetCandidate.authorId, + pushCopyId = outOfNetworkTweetCandidate.pushCopyId, + ntabCopyId = outOfNetworkTweetCandidate.ntabCopyId, + simclusterId = None, + semanticCoreEntityIds = None, + candidateContent = outOfNetworkTweetCandidate.content, + trendId = None + ) + + case userCandidate: PushCandidate with UserCandidate with SocialContextActions => + PushAdaptorUtil.getFrigateNotificationForUser( + userCandidate.commonRecType, + userCandidate.userId, + userCandidate.socialContextActions, + userCandidate.pushCopyId, + userCandidate.ntabCopyId + ) + + case userCandidate: PushCandidate with UserCandidate => + PushAdaptorUtil.getFrigateNotificationForUser( + userCandidate.commonRecType, + userCandidate.userId, + Nil, + userCandidate.pushCopyId, + userCandidate.ntabCopyId + ) + + case tweetCandidate: PushCandidate with TweetCandidate with TweetDetails with SocialContextActions => + PushAdaptorUtil.getFrigateNotificationForTweetWithSocialContextActions( + tweetCandidate.commonRecType, + tweetCandidate.tweetId, + tweetCandidate.socialContextActions, + tweetCandidate.authorId, + tweetCandidate.pushCopyId, + tweetCandidate.ntabCopyId, + candidateContent = tweetCandidate.content, + semanticCoreEntityIds = None, + trendId = None + ) + case pushCandidate: PushCandidate => + FrigateNotification( + commonRecommendationType = pushCandidate.commonRecType, + notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, + pushCopyId = pushCandidate.pushCopyId, + ntabCopyId = pushCandidate.ntabCopyId + ) + + case _ => + statsReceiver + .scope(s"${candidate.commonRecType}").counter("frigate_notification_error").incr() + throw new IllegalStateException("Incorrect candidate type when create FrigateNotification") + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.scala new file mode 100644 index 0000000000..8b737fe672 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.scala @@ -0,0 +1,439 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.channels.common.thriftscala.ApiList +import com.twitter.escherbird.common.thriftscala.Domains +import com.twitter.escherbird.metadata.thriftscala.EntityMegadata +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base._ +import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext +import com.twitter.frigate.magic_events.thriftscala.FanoutEvent +import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason +import com.twitter.frigate.magic_events.thriftscala.TargetID +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model._ +import com.twitter.frigate.pushservice.model.FanoutReasonEntities +import com.twitter.frigate.pushservice.ml.PushMLModelScorer +import com.twitter.frigate.pushservice.model.candidate.CopyIds +import com.twitter.frigate.pushservice.store.EventRequest +import com.twitter.frigate.pushservice.store.UttEntityHydrationStore +import com.twitter.gizmoduck.thriftscala.User +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery +import com.twitter.interests.thriftscala.UserInterests +import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} +import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities +import com.twitter.storehaus.FutureOps +import com.twitter.storehaus.ReadableStore +import com.twitter.strato.client.UserId +import com.twitter.ubs.thriftscala.AudioSpace +import com.twitter.util.Future + +object CandidateHydrationUtil { + + def getAuthorIdFromTweetCandidate(tweetCandidate: TweetCandidate): Option[Long] = { + tweetCandidate match { + case candidate: TweetCandidate with TweetAuthor => + candidate.authorId + case _ => None + } + } + + private def getCandidateAuthorFromUserMap( + tweetCandidate: TweetCandidate, + userMap: Map[Long, User] + ): Option[User] = { + getAuthorIdFromTweetCandidate(tweetCandidate) match { + case Some(id) => + userMap.get(id) + case _ => + None + } + } + + private def getRelationshipMapForInNetworkCandidate( + candidate: RawCandidate with TweetAuthor, + relationshipMap: Map[RelationEdge, Boolean] + ): Map[RelationEdge, Boolean] = { + val relationEdges = + RelationshipUtil.getPreCandidateRelationshipsForInNetworkTweets(candidate).toSet + relationEdges.map { relationEdge => + (relationEdge, relationshipMap(relationEdge)) + }.toMap + } + + private def getTweetCandidateSocialContextUsers( + candidate: RawCandidate with SocialContextActions, + userMap: Map[Long, User] + ): Map[Long, Option[User]] = { + candidate.socialContextUserIds.map { userId => userId -> userMap.get(userId) }.toMap + } + + type TweetWithSocialContextTraits = TweetCandidate with TweetDetails with SocialContextActions + + def getHydratedCandidateForTweetRetweet( + candidate: RawCandidate with TweetWithSocialContextTraits, + userMap: Map[Long, User], + copyIds: CopyIds + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): TweetRetweetPushCandidate = { + new TweetRetweetPushCandidate( + candidate = candidate, + socialContextUserMap = Future.value(getTweetCandidateSocialContextUsers(candidate, userMap)), + author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), + copyIds: CopyIds + ) + } + + def getHydratedCandidateForTweetFavorite( + candidate: RawCandidate with TweetWithSocialContextTraits, + userMap: Map[Long, User], + copyIds: CopyIds + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): TweetFavoritePushCandidate = { + new TweetFavoritePushCandidate( + candidate = candidate, + socialContextUserMap = Future.value(getTweetCandidateSocialContextUsers(candidate, userMap)), + author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), + copyIds = copyIds + ) + } + + def getHydratedCandidateForF1FirstDegreeTweet( + candidate: RawCandidate with F1FirstDegree, + userMap: Map[Long, User], + relationshipMap: Map[RelationEdge, Boolean], + copyIds: CopyIds + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): F1TweetPushCandidate = { + new F1TweetPushCandidate( + candidate = candidate, + author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), + socialGraphServiceResultMap = + getRelationshipMapForInNetworkCandidate(candidate, relationshipMap), + copyIds = copyIds + ) + } + def getHydratedTopicProofTweetCandidate( + candidate: RawCandidate with TopicProofTweetCandidate, + userMap: Map[Long, User], + copyIds: CopyIds + )( + implicit stats: StatsReceiver, + pushMLModelScorer: PushMLModelScorer + ): TopicProofTweetPushCandidate = + new TopicProofTweetPushCandidate( + candidate, + getCandidateAuthorFromUserMap(candidate, userMap), + copyIds + ) + + def getHydratedSubscribedSearchTweetCandidate( + candidate: RawCandidate with SubscribedSearchTweetCandidate, + userMap: Map[Long, User], + copyIds: CopyIds + )( + implicit stats: StatsReceiver, + pushMLModelScorer: PushMLModelScorer + ): SubscribedSearchTweetPushCandidate = + new SubscribedSearchTweetPushCandidate( + candidate, + getCandidateAuthorFromUserMap(candidate, userMap), + copyIds) + + def getHydratedListCandidate( + apiListStore: ReadableStore[Long, ApiList], + candidate: RawCandidate with ListPushCandidate, + copyIds: CopyIds + )( + implicit stats: StatsReceiver, + pushMLModelScorer: PushMLModelScorer + ): ListRecommendationPushCandidate = { + new ListRecommendationPushCandidate(apiListStore, candidate, copyIds) + } + + def getHydratedCandidateForOutOfNetworkTweetCandidate( + candidate: RawCandidate with OutOfNetworkTweetCandidate with TopicCandidate, + userMap: Map[Long, User], + copyIds: CopyIds + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): OutOfNetworkTweetPushCandidate = { + new OutOfNetworkTweetPushCandidate( + candidate: RawCandidate with OutOfNetworkTweetCandidate with TopicCandidate, + author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), + copyIds: CopyIds + ) + } + + def getHydratedCandidateForTripTweetCandidate( + candidate: RawCandidate with OutOfNetworkTweetCandidate with TripCandidate, + userMap: Map[Long, User], + copyIds: CopyIds + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): TripTweetPushCandidate = { + new TripTweetPushCandidate( + candidate: RawCandidate with OutOfNetworkTweetCandidate with TripCandidate, + author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), + copyIds: CopyIds + ) + } + + def getHydratedCandidateForDiscoverTwitterCandidate( + candidate: RawCandidate with DiscoverTwitterCandidate, + copyIds: CopyIds + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): DiscoverTwitterPushCandidate = { + new DiscoverTwitterPushCandidate( + candidate = candidate, + copyIds = copyIds + ) + } + + /** + * /* + * This method can be reusable for hydrating event candidates + **/ + * @param candidate + * @param fanoutMetadataStore + * @param semanticCoreMegadataStore + * @return (hydratedEvent, hydratedFanoutEvent, hydratedSemanticEntityResults, hydratedSemanticCoreMegadata) + */ + private def hydrateMagicFanoutEventCandidate( + candidate: RawCandidate with MagicFanoutEventCandidate, + fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], + semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata] + ): Future[MagicFanoutEventHydratedInfo] = { + + val fanoutEventFut = fanoutMetadataStore.get((candidate.eventId, candidate.pushId)) + + val semanticEntityForQueries: Seq[SemanticEntityForQuery] = { + val semanticCoreEntityIdQueries = candidate.candidateMagicEventsReasons match { + case magicEventsReasons: Seq[MagicEventsReason] => + magicEventsReasons.map(_.reason).collect { + case TargetID.SemanticCoreID(scInterest) => + SemanticEntityForQuery(domainId = scInterest.domainId, entityId = scInterest.entityId) + } + case _ => Seq.empty + } + val eventEntityQuery = SemanticEntityForQuery( + domainId = Domains.EventsEntityService.value, + entityId = candidate.eventId) + semanticCoreEntityIdQueries :+ eventEntityQuery + } + + val semanticEntityResultsFut = FutureOps.mapCollect( + semanticCoreMegadataStore.multiGet(semanticEntityForQueries.toSet) + ) + + Future + .join(fanoutEventFut, semanticEntityResultsFut).map { + case (fanoutEvent, semanticEntityResults) => + MagicFanoutEventHydratedInfo( + fanoutEvent, + semanticEntityResults + ) + case _ => + throw new IllegalArgumentException( + "event candidate hydration errors" + candidate.frigateNotification.toString) + } + } + + def getHydratedCandidateForMagicFanoutNewsEvent( + candidate: RawCandidate with MagicFanoutNewsEventCandidate, + copyIds: CopyIds, + lexServiceStore: ReadableStore[EventRequest, LiveEvent], + fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], + semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], + simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities], + interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], + uttEntityHydrationStore: UttEntityHydrationStore + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): Future[MagicFanoutNewsEventPushCandidate] = { + val magicFanoutEventHydratedInfoFut = hydrateMagicFanoutEventCandidate( + candidate, + fanoutMetadataStore, + semanticCoreMegadataStore + ) + + lazy val simClusterToEntityMappingFut: Future[Map[Int, Option[SimClustersInferredEntities]]] = + Future.collect { + simClusterToEntityStore.multiGet( + FanoutReasonEntities + .from(candidate.candidateMagicEventsReasons.map(_.reason)).simclusterIds.map( + _.clusterId) + ) + } + + Future + .join( + magicFanoutEventHydratedInfoFut, + simClusterToEntityMappingFut + ).map { + case (magicFanoutEventHydratedInfo, simClusterToEntityMapping) => + new MagicFanoutNewsEventPushCandidate( + candidate = candidate, + copyIds = copyIds, + fanoutEvent = magicFanoutEventHydratedInfo.fanoutEvent, + semanticEntityResults = magicFanoutEventHydratedInfo.semanticEntityResults, + simClusterToEntities = simClusterToEntityMapping, + lexServiceStore = lexServiceStore, + interestsLookupStore = interestsLookupStore, + uttEntityHydrationStore = uttEntityHydrationStore + ) + } + } + + def getHydratedCandidateForMagicFanoutSportsEvent( + candidate: RawCandidate + with MagicFanoutSportsEventCandidate + with MagicFanoutSportsScoreInformation, + copyIds: CopyIds, + lexServiceStore: ReadableStore[EventRequest, LiveEvent], + fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], + semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], + interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], + uttEntityHydrationStore: UttEntityHydrationStore + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): Future[MagicFanoutSportsPushCandidate] = { + val magicFanoutEventHydratedInfoFut = hydrateMagicFanoutEventCandidate( + candidate, + fanoutMetadataStore, + semanticCoreMegadataStore + ) + + magicFanoutEventHydratedInfoFut.map { magicFanoutEventHydratedInfo => + new MagicFanoutSportsPushCandidate( + candidate = candidate, + copyIds = copyIds, + fanoutEvent = magicFanoutEventHydratedInfo.fanoutEvent, + semanticEntityResults = magicFanoutEventHydratedInfo.semanticEntityResults, + simClusterToEntities = Map.empty, + lexServiceStore = lexServiceStore, + interestsLookupStore = interestsLookupStore, + uttEntityHydrationStore = uttEntityHydrationStore + ) + } + } + + def getHydratedCandidateForMagicFanoutProductLaunch( + candidate: RawCandidate with MagicFanoutProductLaunchCandidate, + copyIds: CopyIds + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): Future[MagicFanoutProductLaunchPushCandidate] = + Future.value(new MagicFanoutProductLaunchPushCandidate(candidate, copyIds)) + + def getHydratedCandidateForMagicFanoutCreatorEvent( + candidate: RawCandidate with MagicFanoutCreatorEventCandidate, + safeUserStore: ReadableStore[Long, User], + copyIds: CopyIds, + creatorTweetCountStore: ReadableStore[UserId, Int] + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): Future[MagicFanoutCreatorEventPushCandidate] = { + safeUserStore.get(candidate.creatorId).map { hydratedCreatorUser => + new MagicFanoutCreatorEventPushCandidate( + candidate, + hydratedCreatorUser, + copyIds, + creatorTweetCountStore) + } + } + + def getHydratedCandidateForScheduledSpaceSubscriber( + candidate: RawCandidate with ScheduledSpaceSubscriberCandidate, + safeUserStore: ReadableStore[Long, User], + copyIds: CopyIds, + audioSpaceStore: ReadableStore[String, AudioSpace] + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): Future[ScheduledSpaceSubscriberPushCandidate] = { + + candidate.hostId match { + case Some(spaceHostId) => + safeUserStore.get(spaceHostId).map { hydratedHost => + new ScheduledSpaceSubscriberPushCandidate( + candidate = candidate, + hostUser = hydratedHost, + copyIds = copyIds, + audioSpaceStore = audioSpaceStore + ) + } + case _ => + Future.exception( + new IllegalStateException( + "Missing Space Host Id for hydrating ScheduledSpaceSubscriberCandidate")) + } + } + + def getHydratedCandidateForScheduledSpaceSpeaker( + candidate: RawCandidate with ScheduledSpaceSpeakerCandidate, + safeUserStore: ReadableStore[Long, User], + copyIds: CopyIds, + audioSpaceStore: ReadableStore[String, AudioSpace] + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): Future[ScheduledSpaceSpeakerPushCandidate] = { + + candidate.hostId match { + case Some(spaceHostId) => + safeUserStore.get(spaceHostId).map { hydratedHost => + new ScheduledSpaceSpeakerPushCandidate( + candidate = candidate, + hostUser = hydratedHost, + copyIds = copyIds, + audioSpaceStore = audioSpaceStore + ) + } + case _ => + Future.exception( + new RuntimeException( + "Missing Space Host Id for hydrating ScheduledSpaceSpeakerCandidate")) + } + } + + def getHydratedCandidateForTopTweetImpressionsCandidate( + candidate: RawCandidate with TopTweetImpressionsCandidate, + copyIds: CopyIds + )( + implicit stats: StatsReceiver, + pushModelScorer: PushMLModelScorer + ): TopTweetImpressionsPushCandidate = { + new TopTweetImpressionsPushCandidate( + candidate = candidate, + copyIds = copyIds + ) + } + + def isNsfwAccount(user: User, nsfwTokens: Seq[String]): Boolean = { + def hasNsfwToken(str: String): Boolean = nsfwTokens.exists(str.toLowerCase().contains(_)) + + val name = user.profile.map(_.name).getOrElse("") + val screenName = user.profile.map(_.screenName).getOrElse("") + val location = user.profile.map(_.location).getOrElse("") + val description = user.profile.map(_.description).getOrElse("") + val hasNsfwFlag = + user.safety.map(safety => safety.nsfwUser || safety.nsfwAdmin).getOrElse(false) + hasNsfwToken(name) || hasNsfwToken(screenName) || hasNsfwToken(location) || hasNsfwToken( + description) || hasNsfwFlag + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.scala new file mode 100644 index 0000000000..a4802da456 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.scala @@ -0,0 +1,138 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.contentrecommender.thriftscala.MetricTag +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate +import com.twitter.frigate.common.base.SocialContextAction +import com.twitter.frigate.common.base.SocialContextActions +import com.twitter.frigate.common.base.TargetInfo +import com.twitter.frigate.common.base.TargetUser +import com.twitter.frigate.common.base.TopicProofTweetCandidate +import com.twitter.frigate.common.base.TweetAuthorDetails +import com.twitter.frigate.common.candidate.TargetABDecider +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.params.CrtGroupEnum +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.CommonRecommendationType.TripGeoTweet +import com.twitter.frigate.thriftscala.CommonRecommendationType.TripHqTweet +import com.twitter.frigate.thriftscala.{SocialContextAction => TSocialContextAction} +import com.twitter.util.Future + +object CandidateUtil { + private val mrTwistlyMetricTags = + Seq(MetricTag.PushOpenOrNtabClick, MetricTag.RequestHealthFilterPushOpenBasedTweetEmbedding) + + def getSocialContextActionsFromCandidate(candidate: RawCandidate): Seq[TSocialContextAction] = { + candidate match { + case candidateWithSocialContex: RawCandidate with SocialContextActions => + candidateWithSocialContex.socialContextActions.map { scAction => + TSocialContextAction( + scAction.userId, + scAction.timestampInMillis, + scAction.tweetId + ) + } + case _ => Seq.empty + } + } + + /** + * Ranking Social Context based on the Real Graph weight + * @param socialContextActions Sequence of Social Context Actions + * @param seedsWithWeight Real Graph map consisting of User ID as key and RG weight as the value + * @param defaultToRecency Boolean to represent if we should use the timestamp of the SC to rank + * @return Returns the ranked sequence of SC Actions + */ + def getRankedSocialContext( + socialContextActions: Seq[SocialContextAction], + seedsWithWeight: Future[Option[Map[Long, Double]]], + defaultToRecency: Boolean + ): Future[Seq[SocialContextAction]] = { + seedsWithWeight.map { + case Some(followingsMap) => + socialContextActions.sortBy { action => -followingsMap.getOrElse(action.userId, 0.0) } + case _ => + if (defaultToRecency) socialContextActions.sortBy(-_.timestampInMillis) + else socialContextActions + } + } + + def shouldApplyHealthQualityFiltersForPrerankingPredicates( + candidate: TweetAuthorDetails with TargetInfo[TargetUser with TargetABDecider] + )( + implicit stats: StatsReceiver + ): Future[Boolean] = { + candidate.tweetAuthor.map { + case Some(user) => + val numFollowers: Double = user.counts.map(_.followers.toDouble).getOrElse(0.0) + numFollowers < candidate.target + .params(PushFeatureSwitchParams.NumFollowerThresholdForHealthAndQualityFiltersPreranking) + case _ => true + } + } + + def shouldApplyHealthQualityFilters( + candidate: PushCandidate + )( + implicit stats: StatsReceiver + ): Boolean = { + val numFollowers = + candidate.numericFeatures.getOrElse("RecTweetAuthor.User.ActiveFollowers", 0.0) + numFollowers < candidate.target + .params(PushFeatureSwitchParams.NumFollowerThresholdForHealthAndQualityFilters) + } + + def useAggressiveHealthThresholds(cand: PushCandidate): Boolean = + isMrTwistlyCandidate(cand) || + (cand.commonRecType == CommonRecommendationType.GeoPopTweet && cand.target.params( + PushFeatureSwitchParams.PopGeoTweetEnableAggressiveThresholds)) + + def isMrTwistlyCandidate(cand: PushCandidate): Boolean = + cand match { + case oonCandidate: PushCandidate with OutOfNetworkTweetCandidate => + oonCandidate.tagsCR + .getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty && oonCandidate.tagsCR + .map(_.toSet.size).getOrElse(0) == 1 + case oonCandidate: PushCandidate with TopicProofTweetCandidate + if cand.target.params(PushFeatureSwitchParams.EnableHealthFiltersForTopicProofTweet) => + oonCandidate.tagsCR + .getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty && oonCandidate.tagsCR + .map(_.toSet.size).getOrElse(0) == 1 + case _ => false + } + + def getTagsCRCount(cand: PushCandidate): Double = + cand match { + case oonCandidate: PushCandidate with OutOfNetworkTweetCandidate => + oonCandidate.tagsCR.map(_.toSet.size).getOrElse(0).toDouble + case oonCandidate: PushCandidate with TopicProofTweetCandidate + if cand.target.params(PushFeatureSwitchParams.EnableHealthFiltersForTopicProofTweet) => + oonCandidate.tagsCR.map(_.toSet.size).getOrElse(0).toDouble + case _ => 0.0 + } + + def isRelatedToMrTwistlyCandidate(cand: PushCandidate): Boolean = + cand match { + case oonCandidate: PushCandidate with OutOfNetworkTweetCandidate => + oonCandidate.tagsCR.getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty + case oonCandidate: PushCandidate with TopicProofTweetCandidate + if cand.target.params(PushFeatureSwitchParams.EnableHealthFiltersForTopicProofTweet) => + oonCandidate.tagsCR.getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty + case _ => false + } + + def getCrtGroup(commonRecType: CommonRecommendationType): CrtGroupEnum.Value = { + commonRecType match { + case crt if RecTypes.twistlyTweets(crt) => CrtGroupEnum.Twistly + case crt if RecTypes.frsTypes(crt) => CrtGroupEnum.Frs + case crt if RecTypes.f1RecTypes(crt) => CrtGroupEnum.F1 + case crt if crt == TripGeoTweet || crt == TripHqTweet => CrtGroupEnum.Trip + case crt if RecTypes.TopicTweetTypes(crt) => CrtGroupEnum.Topic + case crt if RecTypes.isGeoPopTweetType(crt) => CrtGroupEnum.GeoPop + case _ => CrtGroupEnum.Other + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.scala new file mode 100644 index 0000000000..95208c35e7 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.scala @@ -0,0 +1,448 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} +import com.twitter.ibis2.lib.util.JsonMarshal +import com.twitter.util.Future +import com.twitter.util.Time + +object CopyUtil { + + /** + * Get a list of history feature copy alone with metadata in the look back period, the metadata + * can be used to calculate number of copy pushed after the current feature copy + * @param candidate the candidate to be pushed to the user + * @return Future[Seq((..,))], which is a seq of the history FEATURE copy along with + * metadata within the look back period. In the tuple, the 4 elements represents: + * 1. Timestamp of the past feature copy + * 2. Option[Seq()] of copy feature names of the past copy + * 3. Index of the particular feature copy in look back history if normal copy presents + */ + private def getPastCopyFeaturesList( + candidate: PushCandidate + ): Future[Seq[(Time, Option[Seq[String]], Int)]] = { + val target = candidate.target + + target.history.map { targetHistory => + val historyLookbackDuration = target.params(FS.CopyFeaturesHistoryLookbackDuration) + val notificationHistoryInLookbackDuration = targetHistory.sortedHistory + .takeWhile { + case (notifTimestamp, _) => historyLookbackDuration.ago < notifTimestamp + } + notificationHistoryInLookbackDuration.zipWithIndex + .filter { + case ((_, notification), _) => + notification.copyFeatures match { + case Some(copyFeatures) => copyFeatures.nonEmpty + case _ => false + } + } + .collect { + case ((timestamp, notification), notificationIndex) => + (timestamp, notification.copyFeatures, notificationIndex) + } + } + } + + private def getPastCopyFeaturesListForF1( + candidate: PushCandidate + ): Future[Seq[(Time, Option[Seq[String]], Int)]] = { + val target = candidate.target + target.history.map { targetHistory => + val historyLookbackDuration = target.params(FS.CopyFeaturesHistoryLookbackDuration) + val notificationHistoryInLookbackDuration = targetHistory.sortedHistory + .takeWhile { + case (notifTimestamp, _) => historyLookbackDuration.ago < notifTimestamp + } + notificationHistoryInLookbackDuration.zipWithIndex + .filter { + case ((_, notification), _) => + notification.copyFeatures match { + case Some(copyFeatures) => + RecTypes.isF1Type(notification.commonRecommendationType) && copyFeatures.nonEmpty + case _ => false + } + } + .collect { + case ((timestamp, notification), notificationIndex) => + (timestamp, notification.copyFeatures, notificationIndex) + } + } + } + + private def getPastCopyFeaturesListForOON( + candidate: PushCandidate + ): Future[Seq[(Time, Option[Seq[String]], Int)]] = { + val target = candidate.target + target.history.map { targetHistory => + val historyLookbackDuration = target.params(FS.CopyFeaturesHistoryLookbackDuration) + val notificationHistoryInLookbackDuration = targetHistory.sortedHistory + .takeWhile { + case (notifTimestamp, _) => historyLookbackDuration.ago < notifTimestamp + } + notificationHistoryInLookbackDuration.zipWithIndex + .filter { + case ((_, notification), _) => + notification.copyFeatures match { + case Some(copyFeatures) => + !RecTypes.isF1Type(notification.commonRecommendationType) && copyFeatures.nonEmpty + + case _ => false + } + } + .collect { + case ((timestamp, notification), notificationIndex) => + (timestamp, notification.copyFeatures, notificationIndex) + } + } + } + private def getEmojiFeaturesMap( + candidate: PushCandidate, + copyFeatureHistory: Seq[(Time, Option[Seq[String]], Int)], + lastHTLVisitTimestamp: Option[Long], + stats: StatsReceiver + ): Map[String, String] = { + val (emojiFatigueDuration, emojiFatigueNumOfPushes) = { + if (RecTypes.isF1Type(candidate.commonRecType)) { + ( + candidate.target.params(FS.F1EmojiCopyFatigueDuration), + candidate.target.params(FS.F1EmojiCopyNumOfPushesFatigue)) + } else { + ( + candidate.target.params(FS.OonEmojiCopyFatigueDuration), + candidate.target.params(FS.OonEmojiCopyNumOfPushesFatigue)) + } + } + + val scopedStats = stats + .scope("getEmojiFeaturesMap").scope(candidate.commonRecType.toString).scope( + emojiFatigueDuration.toString) + val addedEmojiCopyFeature = scopedStats.counter("added_emoji") + val fatiguedEmojiCopyFeature = scopedStats.counter("no_emoji") + + val copyFeatureType = PushConstants.EmojiFeatureNameForIbis2ModelValues + + val durationFatigueCarryFunc = () => + isUnderDurationFatigue(copyFeatureHistory, copyFeatureType, emojiFatigueDuration) + + val enableHTLBasedFatigueBasicRule = candidate.target.params(FS.EnableHTLBasedFatigueBasicRule) + val minDuration = candidate.target.params(FS.MinFatigueDurationSinceLastHTLVisit) + val lastHTLVisitBasedNonFatigueWindow = + candidate.target.params(FS.LastHTLVisitBasedNonFatigueWindow) + val htlBasedCopyFatigueCarryFunc = () => + isUnderHTLBasedFatigue(lastHTLVisitTimestamp, minDuration, lastHTLVisitBasedNonFatigueWindow) + + val isUnderFatigue = getIsUnderFatigue( + Seq( + (durationFatigueCarryFunc, true), + (htlBasedCopyFatigueCarryFunc, enableHTLBasedFatigueBasicRule), + ), + scopedStats + ) + + if (!isUnderFatigue) { + addedEmojiCopyFeature.incr() + Map(PushConstants.EmojiFeatureNameForIbis2ModelValues -> "true") + } else { + fatiguedEmojiCopyFeature.incr() + Map.empty[String, String] + } + } + + private def getTargetFeaturesMap( + candidate: PushCandidate, + copyFeatureHistory: Seq[(Time, Option[Seq[String]], Int)], + lastHTLVisitTimestamp: Option[Long], + stats: StatsReceiver + ): Map[String, String] = { + val targetFatigueDuration = { + if (RecTypes.isF1Type(candidate.commonRecType)) { + candidate.target.params(FS.F1TargetCopyFatigueDuration) + } else { + candidate.target.params(FS.OonTargetCopyFatigueDuration) + } + } + + val scopedStats = stats + .scope("getTargetFeaturesMap").scope(candidate.commonRecType.toString).scope( + targetFatigueDuration.toString) + val addedTargetCopyFeature = scopedStats.counter("added_target") + val fatiguedTargetCopyFeature = scopedStats.counter("no_target") + + val featureCopyType = PushConstants.TargetFeatureNameForIbis2ModelValues + val durationFatigueCarryFunc = () => + isUnderDurationFatigue(copyFeatureHistory, featureCopyType, targetFatigueDuration) + + val enableHTLBasedFatigueBasicRule = candidate.target.params(FS.EnableHTLBasedFatigueBasicRule) + val minDuration = candidate.target.params(FS.MinFatigueDurationSinceLastHTLVisit) + val lastHTLVisitBasedNonFatigueWindow = + candidate.target.params(FS.LastHTLVisitBasedNonFatigueWindow) + val htlBasedCopyFatigueCarryFunc = () => + isUnderHTLBasedFatigue(lastHTLVisitTimestamp, minDuration, lastHTLVisitBasedNonFatigueWindow) + + val isUnderFatigue = getIsUnderFatigue( + Seq( + (durationFatigueCarryFunc, true), + (htlBasedCopyFatigueCarryFunc, enableHTLBasedFatigueBasicRule), + ), + scopedStats + ) + + if (!isUnderFatigue) { + addedTargetCopyFeature.incr() + Map(PushConstants.TargetFeatureNameForIbis2ModelValues -> "true") + } else { + + fatiguedTargetCopyFeature.incr() + Map.empty[String, String] + } + } + + type FatigueRuleFlag = Boolean + type FatigueRuleFunc = () => Boolean + + def getIsUnderFatigue( + fatigueRulesWithFlags: Seq[(FatigueRuleFunc, FatigueRuleFlag)], + statsReceiver: StatsReceiver, + ): Boolean = { + val defaultFatigue = true + val finalFatigueRes = + fatigueRulesWithFlags.zipWithIndex.foldLeft(defaultFatigue)( + (fatigueSoFar, fatigueRuleFuncWithFlagAndIndex) => { + val ((fatigueRuleFunc, flag), index) = fatigueRuleFuncWithFlagAndIndex + val funcScopedStats = statsReceiver.scope(s"fatigueFunction${index}") + if (flag) { + val shouldFatigueForTheRule = fatigueRuleFunc() + funcScopedStats.scope(s"eval_${shouldFatigueForTheRule}").counter().incr() + val f = fatigueSoFar && shouldFatigueForTheRule + f + } else { + fatigueSoFar + } + }) + statsReceiver.scope(s"final_fatigue_${finalFatigueRes}").counter().incr() + finalFatigueRes + } + + private def isUnderDurationFatigue( + copyFeatureHistory: Seq[(Time, Option[Seq[String]], Int)], + copyFeatureType: String, + fatigueDuration: com.twitter.util.Duration, + ): Boolean = { + copyFeatureHistory.exists { + case (notifTimestamp, Some(copyFeatures), _) if copyFeatures.contains(copyFeatureType) => + notifTimestamp > fatigueDuration.ago + case _ => false + } + } + + private def isUnderHTLBasedFatigue( + lastHTLVisitTimestamp: Option[Long], + minDurationSinceLastHTLVisit: com.twitter.util.Duration, + lastHTLVisitBasedNonFatigueWindow: com.twitter.util.Duration, + ): Boolean = { + val lastHTLVisit = lastHTLVisitTimestamp.map(t => Time.fromMilliseconds(t)).getOrElse(Time.now) + val first = Time.now < (lastHTLVisit + minDurationSinceLastHTLVisit) + val second = + Time.now > (lastHTLVisit + minDurationSinceLastHTLVisit + lastHTLVisitBasedNonFatigueWindow) + first || second + } + + def getOONCBasedFeature( + candidate: PushCandidate, + stats: StatsReceiver + ): Future[Map[String, String]] = { + val target = candidate.target + val metric = stats.scope("getOONCBasedFeature") + if (target.params(FS.EnableOONCBasedCopy)) { + candidate.mrWeightedOpenOrNtabClickRankingProbability.map { + case Some(score) if score >= target.params(FS.HighOONCThresholdForCopy) => + metric.counter("high_OONC").incr() + metric.counter(FS.HighOONCTweetFormat.toString).incr() + Map( + "whole_template" -> JsonMarshal.toJson( + Map( + target.params(FS.HighOONCTweetFormat).toString -> true + ))) + case Some(score) if score <= target.params(FS.LowOONCThresholdForCopy) => + metric.counter("low_OONC").incr() + metric.counter(FS.LowOONCThresholdForCopy.toString).incr() + Map( + "whole_template" -> JsonMarshal.toJson( + Map( + target.params(FS.LowOONCTweetFormat).toString -> true + ))) + case _ => + metric.counter("not_in_OONC_range").incr() + Map.empty[String, String] + } + } else { + Future.value(Map.empty[String, String]) + } + } + + def getCopyFeatures( + candidate: PushCandidate, + stats: StatsReceiver, + ): Future[Map[String, String]] = { + if (candidate.target.isLoggedOutUser) { + Future.value(Map.empty[String, String]) + } else { + val featureMaps = getCopyBodyFeatures(candidate, stats) + for { + titleFeat <- getCopyTitleFeatures(candidate, stats) + nsfwFeat <- getNsfwCopyFeatures(candidate, stats) + ooncBasedFeature <- getOONCBasedFeature(candidate, stats) + } yield { + titleFeat ++ featureMaps ++ nsfwFeat ++ ooncBasedFeature + } + } + } + + private def getCopyTitleFeatures( + candidate: PushCandidate, + stats: StatsReceiver + ): Future[Map[String, String]] = { + val scopedStats = stats.scope("CopyUtil").scope("getCopyTitleFeatures") + + val target = candidate.target + + if ((RecTypes.isSimClusterBasedType(candidate.commonRecType) && target.params( + FS.EnableCopyFeaturesForOon)) || (RecTypes.isF1Type(candidate.commonRecType) && target + .params(FS.EnableCopyFeaturesForF1))) { + + val enableTargetAndEmojiSplitFatigue = target.params(FS.EnableTargetAndEmojiSplitFatigue) + val isTargetF1Type = RecTypes.isF1Type(candidate.commonRecType) + + val copyFeatureHistoryFuture = if (enableTargetAndEmojiSplitFatigue && isTargetF1Type) { + getPastCopyFeaturesListForF1(candidate) + } else if (enableTargetAndEmojiSplitFatigue && !isTargetF1Type) { + getPastCopyFeaturesListForOON(candidate) + } else { + getPastCopyFeaturesList(candidate) + } + + Future + .join( + copyFeatureHistoryFuture, + target.lastHTLVisitTimestamp, + ).map { + case (copyFeatureHistory, lastHTLVisitTimestamp) => + val emojiFeatures = { + if ((RecTypes.isF1Type(candidate.commonRecType) && target.params( + FS.EnableEmojiInF1Copy)) + || RecTypes.isSimClusterBasedType(candidate.commonRecType) && target.params( + FS.EnableEmojiInOonCopy)) { + getEmojiFeaturesMap( + candidate, + copyFeatureHistory, + lastHTLVisitTimestamp, + scopedStats) + } else Map.empty[String, String] + } + + val targetFeatures = { + if ((RecTypes.isF1Type(candidate.commonRecType) && target.params( + FS.EnableTargetInF1Copy)) || (RecTypes.isSimClusterBasedType( + candidate.commonRecType) && target.params(FS.EnableTargetInOonCopy))) { + getTargetFeaturesMap( + candidate, + copyFeatureHistory, + lastHTLVisitTimestamp, + scopedStats) + } else Map.empty[String, String] + } + + val baseCopyFeaturesMap = + if (emojiFeatures.nonEmpty || targetFeatures.nonEmpty) + Map(PushConstants.EnableCopyFeaturesForIbis2ModelValues -> "true") + else Map.empty[String, String] + baseCopyFeaturesMap ++ emojiFeatures ++ targetFeatures + case _ => + Map.empty[String, String] + } + } else Future.value(Map.empty[String, String]) + } + + private def getCopyBodyTruncateFeatures( + candidate: PushCandidate, + ): Map[String, String] = { + if (candidate.target.params(FS.EnableIosCopyBodyTruncate)) { + Map("enable_body_truncate_ios" -> "true") + } else { + Map.empty[String, String] + } + } + + private def getNsfwCopyFeatures( + candidate: PushCandidate, + stats: StatsReceiver + ): Future[Map[String, String]] = { + val scopedStats = stats.scope("CopyUtil").scope("getNsfwCopyBodyFeatures") + val hasNsfwScoreF1Counter = scopedStats.counter("f1_has_nsfw_score") + val hasNsfwScoreOonCounter = scopedStats.counter("oon_has_nsfw_score") + val noNsfwScoreCounter = scopedStats.counter("no_nsfw_score") + val nsfwScoreF1 = scopedStats.stat("f1_nsfw_score") + val nsfwScoreOon = scopedStats.stat("oon_nsfw_score") + val isNsfwF1Counter = scopedStats.counter("is_f1_nsfw") + val isNsfwOonCounter = scopedStats.counter("is_oon_nsfw") + + val target = candidate.target + val nsfwScoreFut = if (target.params(FS.EnableNsfwCopy)) { + candidate.mrNsfwScore + } else Future.None + + nsfwScoreFut.map { + case Some(nsfwScore) => + if (RecTypes.isF1Type(candidate.commonRecType)) { + hasNsfwScoreF1Counter.incr() + nsfwScoreF1.add(nsfwScore.toFloat * 10000) + if (nsfwScore > target.params(FS.NsfwScoreThresholdForF1Copy)) { + isNsfwF1Counter.incr() + Map("is_f1_nsfw" -> "true") + } else { + Map.empty[String, String] + } + } else if (RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType)) { + nsfwScoreOon.add(nsfwScore.toFloat * 10000) + hasNsfwScoreOonCounter.incr() + if (nsfwScore > target.params(FS.NsfwScoreThresholdForOONCopy)) { + isNsfwOonCounter.incr() + Map("is_oon_nsfw" -> "true") + } else { + Map.empty[String, String] + } + } else { + Map.empty[String, String] + } + case _ => + noNsfwScoreCounter.incr() + Map.empty[String, String] + } + } + + private def getCopyBodyFeatures( + candidate: PushCandidate, + stats: StatsReceiver + ): Map[String, String] = { + val target = candidate.target + val scopedStats = stats.scope("CopyUtil").scope("getCopyBodyFeatures") + + val copyBodyFeatures = { + if (RecTypes.isF1Type(candidate.commonRecType) && target.params(FS.EnableF1CopyBody)) { + scopedStats.counter("f1BodyExpEnabled").incr() + Map(PushConstants.CopyBodyExpIbisModelValues -> "true") + } else if (RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) && target.params( + FS.EnableOONCopyBody)) { + scopedStats.counter("oonBodyExpEnabled").incr() + Map(PushConstants.CopyBodyExpIbisModelValues -> "true") + } else + Map.empty[String, String] + } + val copyBodyTruncateFeatures = getCopyBodyTruncateFeatures(candidate) + copyBodyFeatures ++ copyBodyTruncateFeatures + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.scala new file mode 100644 index 0000000000..34d5b9bae6 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.scala @@ -0,0 +1,92 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.frigate.common.store.deviceinfo.DeviceInfo +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.EnableRuxLandingPage +import com.twitter.frigate.pushservice.params.PushParams.EnableRuxLandingPageAndroidParam +import com.twitter.frigate.pushservice.params.PushParams.EnableRuxLandingPageIOSParam +import com.twitter.frigate.pushservice.params.PushParams.RuxLandingPageExperimentKeyAndroidParam +import com.twitter.frigate.pushservice.params.PushParams.RuxLandingPageExperimentKeyIOSParam +import com.twitter.frigate.pushservice.params.PushParams.ShowRuxLandingPageAsModalOnIOS +import com.twitter.rux.common.context.thriftscala.MagicRecsNTabTweet +import com.twitter.rux.common.context.thriftscala.MagicRecsPushTweet +import com.twitter.rux.common.context.thriftscala.RuxContext +import com.twitter.rux.common.context.thriftscala.Source +import com.twitter.rux.common.encode.RuxContextEncoder + +/** + * This class provides utility functions for email landing page for push + */ +object EmailLandingPageExperimentUtil { + val ruxCxtEncoder = new RuxContextEncoder() + + def getIbis2ModelValue( + deviceInfoOpt: Option[DeviceInfo], + target: Target, + tweetId: Long + ): Map[String, String] = { + val enable = enablePushEmailLanding(deviceInfoOpt, target) + if (enable) { + val ruxCxt = if (deviceInfoOpt.exists(_.isRuxLandingPageEligible)) { + val encodedCxt = getRuxContext(tweetId, target, deviceInfoOpt) + Map("rux_cxt" -> encodedCxt) + } else Map.empty[String, String] + val enableModal = if (showModalForIOS(deviceInfoOpt, target)) { + Map("enable_modal" -> "true") + } else Map.empty[String, String] + + Map("land_on_email_landing_page" -> "true") ++ ruxCxt ++ enableModal + } else Map.empty[String, String] + } + + def createNTabRuxLandingURI(screenName: String, tweetId: Long): String = { + val encodedCxt = + ruxCxtEncoder.encode(RuxContext(Some(Source.MagicRecsNTabTweet(MagicRecsNTabTweet(tweetId))))) + s"$screenName/status/${tweetId.toString}?cxt=$encodedCxt" + } + + private def getRuxContext( + tweetId: Long, + target: Target, + deviceInfoOpt: Option[DeviceInfo] + ): String = { + val isDeviceIOS = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) + val isDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) + val keyOpt = if (isDeviceIOS) { + target.params(RuxLandingPageExperimentKeyIOSParam) + } else if (isDeviceAndroid) { + target.params(RuxLandingPageExperimentKeyAndroidParam) + } else None + val context = RuxContext(Some(Source.MagicRecsTweet(MagicRecsPushTweet(tweetId))), None, keyOpt) + ruxCxtEncoder.encode(context) + } + + private def enablePushEmailLanding( + deviceInfoOpt: Option[DeviceInfo], + target: Target + ): Boolean = + deviceInfoOpt.exists(deviceInfo => + if (deviceInfo.isEmailLandingPageEligible) { + val isRuxLandingPageEnabled = target.params(EnableRuxLandingPage) + isRuxLandingPageEnabled && isRuxLandingEnabledBasedOnDeviceInfo(deviceInfoOpt, target) + } else false) + + private def showModalForIOS(deviceInfoOpt: Option[DeviceInfo], target: Target): Boolean = { + deviceInfoOpt.exists { deviceInfo => + deviceInfo.isRuxLandingPageAsModalEligible && target.params(ShowRuxLandingPageAsModalOnIOS) + } + } + + private def isRuxLandingEnabledBasedOnDeviceInfo( + deviceInfoOpt: Option[DeviceInfo], + target: Target + ): Boolean = { + val isDeviceIOS = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) + val isDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) + if (isDeviceIOS) { + target.params(EnableRuxLandingPageIOSParam) + } else if (isDeviceAndroid) { + target.params(EnableRuxLandingPageAndroidParam) + } else true + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.scala new file mode 100644 index 0000000000..a2721873e6 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.scala @@ -0,0 +1,12 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.Counter + +object FunctionalUtil { + def incr[T](counter: Counter): T => T = { x => + { + counter.incr() + x + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.scala new file mode 100644 index 0000000000..de1108f562 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.scala @@ -0,0 +1,55 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.CommonRecommendationType._ + +object IbisScribeTargets { + val User2 = "magic_rec_user_2" + val User4 = "magic_rec_user_4" + val Tweet2 = "magic_rec_tweet_2" + val Tweet4 = "magic_rec_tweet_4" + val Tweet5 = "magic_rec_tweet_5" + val Tweet9 = "magic_rec_tweet_9" + val Tweet10 = "magic_rec_tweet_10" + val Tweet11 = "magic_rec_tweet_11" + val Tweet12 = "magic_rec_tweet_12" + val Tweet16 = "magic_rec_tweet_16" + val Hashtag = "magic_rec_hashtag" + val UnreadBadgeCount17 = "magic_rec_unread_badge_count_17" + val Highlights = "highlights" + val TweetAnalytics = "magic_rec_tweet_analytics" + val Untracked = "untracked" + + def crtToScribeTarget(crt: CommonRecommendationType): String = crt match { + case UserFollow => + User2 + case HermitUser => + User4 + case TweetRetweet | TweetFavorite => + Tweet2 + case TweetRetweetPhoto | TweetFavoritePhoto => + Tweet4 + case TweetRetweetVideo | TweetFavoriteVideo => + Tweet5 + case UrlTweetLanding => + Tweet9 + case F1FirstdegreeTweet | F1FirstdegreePhoto | F1FirstdegreeVideo => + Tweet10 + case AuthorTargetingTweet => + Tweet11 + case PeriscopeShare => + Tweet12 + case CommonRecommendationType.Highlights => + Highlights + case HashtagTweet | HashtagTweetRetweet => + Hashtag + case PinnedTweet => + Tweet16 + case UnreadBadgeCount => + UnreadBadgeCount17 + case TweetImpressions => + TweetAnalytics + case _ => + Untracked + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.scala new file mode 100644 index 0000000000..2900b7418c --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.scala @@ -0,0 +1,219 @@ +package com.twitter.frigate.pushservice.util + +import com.google.common.io.BaseEncoding +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.InlineActionsEnum +import com.twitter.frigate.pushservice.params.PushParams +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.ibis2.lib.util.JsonMarshal +import com.twitter.notifications.platform.thriftscala._ +import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse +import com.twitter.scrooge.BinaryThriftStructSerializer +import com.twitter.util.Future + +/** + * This class provides utility functions for inline action for push + */ +object InlineActionUtil { + + def scopedStats(statsReceiver: StatsReceiver): StatsReceiver = + statsReceiver.scope(getClass.getSimpleName) + + /** + * Util function to build web inline actions for Ibis + * @param actions list of inline actions to be hydrated depending on the CRT + * @param enableForDesktopWeb if web inline actions should be shown on desktop RWeb, for experimentation purpose + * @param enableForMobileWeb if web inline actions should be shwon on mobile RWeb, for experimentation purpose + * @return Params for web inline actions to be consumed by `smart.inline.actions.web.mustache` in Ibis + */ + def getGeneratedTweetInlineActionsForWeb( + actions: Seq[InlineActionsEnum.Value], + enableForDesktopWeb: Boolean, + enableForMobileWeb: Boolean + ): Map[String, String] = { + if (!enableForDesktopWeb && !enableForMobileWeb) { + Map.empty + } else { + val inlineActions = buildEnrichedInlineActionsMap(actions) ++ Map( + "enable_for_desktop_web" -> enableForDesktopWeb.toString, + "enable_for_mobile_web" -> enableForMobileWeb.toString + ) + Map( + "inline_action_details_web" -> JsonMarshal.toJson(inlineActions), + ) + } + } + + def getGeneratedTweetInlineActionsV1( + actions: Seq[InlineActionsEnum.Value] + ): Map[String, String] = { + val inlineActions = buildEnrichedInlineActionsMap(actions) + Map( + "inline_action_details" -> JsonMarshal.toJson(inlineActions) + ) + } + + private def buildEnrichedInlineActionsMap( + actions: Seq[InlineActionsEnum.Value] + ): Map[String, Seq[Map[String, Any]]] = { + Map( + "actions" -> actions + .map(_.toString.toLowerCase) + .zipWithIndex + .map { + case (a: String, i: Int) => + Map("action" -> a) ++ Map( + s"use_${a}_stringcenter_key" -> true, + "last" -> (i == (actions.length - 1)) + ) + }.seq + ) + } + + def getGeneratedTweetInlineActionsV2( + actions: Seq[InlineActionsEnum.Value] + ): Map[String, String] = { + val v2CustomActions = actions + .map { + case InlineActionsEnum.Favorite => + NotificationCustomAction( + Some("mr_inline_favorite_title"), + CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Favorite)) + ) + case InlineActionsEnum.Follow => + NotificationCustomAction( + Some("mr_inline_follow_title"), + CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Follow))) + case InlineActionsEnum.Reply => + NotificationCustomAction( + Some("mr_inline_reply_title"), + CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Reply))) + case InlineActionsEnum.Retweet => + NotificationCustomAction( + Some("mr_inline_retweet_title"), + CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Retweet))) + case _ => + NotificationCustomAction( + Some("mr_inline_favorite_title"), + CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Favorite)) + ) + } + val notifications = NotificationCustomActions(v2CustomActions) + Map("serialized_inline_actions_v2" -> serializeActionsToBase64(notifications)) + } + + def getDislikeInlineAction( + candidate: PushCandidate, + ntabResponse: CreateGenericNotificationResponse + ): Option[NotificationCustomAction] = { + ntabResponse.successKey.map(successKey => { + val urlParams = Map[String, String]( + "answer" -> "dislike", + "notification_hash" -> successKey.hashKey.toString, + "upstream_uid" -> candidate.impressionId, + "notification_timestamp" -> successKey.timestampMillis.toString + ) + val urlParamsString = urlParams.map(kvp => f"${kvp._1}=${kvp._2}").mkString("&") + + val httpPostRequest = HttpRequest.PostRequest( + PostRequest(url = f"/2/notifications/feedback.json?$urlParamsString", bodyParams = None)) + val httpRequestAction = HttpRequestAction( + httpRequest = httpPostRequest, + scribeAction = Option("dislike_scribe_action"), + isAuthorizationRequired = Option(true), + isDestructive = Option(false), + undoable = None + ) + val dislikeAction = CustomActionData.HttpRequestAction(httpRequestAction) + NotificationCustomAction(title = Option("mr_inline_dislike_title"), action = dislikeAction) + }) + } + + /** + * Given a serialized inline action v2, update the action at index to the given new action. + * If given index is bigger than current action length, append the given inline action at the end. + * @param serialized_inline_actions_v2 the original action in serialized version + * @param actionOption an Option of the new action to replace the old one + * @param index the position where the old action will be replaced + * @return a new serialized inline action v2 + */ + def patchInlineActionAtPosition( + serialized_inline_actions_v2: String, + actionOption: Option[NotificationCustomAction], + index: Int + ): String = { + val originalActions: Seq[NotificationCustomAction] = deserializeActionsFromString( + serialized_inline_actions_v2).actions + val newActions = actionOption match { + case Some(action) if index >= originalActions.size => originalActions ++ Seq(action) + case Some(action) => originalActions.updated(index, action) + case _ => originalActions + } + serializeActionsToBase64(NotificationCustomActions(newActions)) + } + + /** + * Return list of available inline actions for ibis2 model + */ + def getGeneratedTweetInlineActions( + target: Target, + statsReceiver: StatsReceiver, + actions: Seq[InlineActionsEnum.Value], + ): Map[String, String] = { + val scopedStatsReceiver = scopedStats(statsReceiver) + val useV1 = target.params(PushFeatureSwitchParams.UseInlineActionsV1) + val useV2 = target.params(PushFeatureSwitchParams.UseInlineActionsV2) + if (useV1 && useV2) { + scopedStatsReceiver.counter("use_v1_and_use_v2").incr() + getGeneratedTweetInlineActionsV1(actions) ++ getGeneratedTweetInlineActionsV2(actions) + } else if (useV1 && !useV2) { + scopedStatsReceiver.counter("only_use_v1").incr() + getGeneratedTweetInlineActionsV1(actions) + } else if (!useV1 && useV2) { + scopedStatsReceiver.counter("only_use_v2").incr() + getGeneratedTweetInlineActionsV2(actions) + } else { + scopedStatsReceiver.counter("use_neither_v1_nor_v2").incr() + Map.empty[String, String] + } + } + + /** + * Return Tweet inline action ibis2 model values after applying experiment logic + */ + def getTweetInlineActionValue(target: Target): Future[Map[String, String]] = { + if (target.isLoggedOutUser) { + Future( + Map( + "show_inline_action" -> "false" + ) + ) + } else { + val showInlineAction: Boolean = target.params(PushParams.MRAndroidInlineActionOnPushCopyParam) + Future( + Map( + "show_inline_action" -> s"$showInlineAction" + ) + ) + } + } + + private val binaryThriftStructSerializer: BinaryThriftStructSerializer[ + NotificationCustomActions + ] = BinaryThriftStructSerializer.apply(NotificationCustomActions) + private val base64Encoding = BaseEncoding.base64() + + def serializeActionsToBase64(notificationCustomActions: NotificationCustomActions): String = { + val actionsAsByteArray: Array[Byte] = + binaryThriftStructSerializer.toBytes(notificationCustomActions) + base64Encoding.encode(actionsAsByteArray) + } + + def deserializeActionsFromString(serializedInlineActionV2: String): NotificationCustomActions = { + val actionAsByteArray = base64Encoding.decode(serializedInlineActionV2) + binaryThriftStructSerializer.fromBytes(actionAsByteArray) + } + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.scala new file mode 100644 index 0000000000..a3a3ecf50f --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.scala @@ -0,0 +1,52 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate + +object MediaAnnotationsUtil { + + val mediaIdToCategoryMapping = Map("0" -> "0") + + val nudityCategoryId = "0" + val beautyCategoryId = "0" + val singlePersonCategoryId = "0" + val sensitiveMediaCategoryFeatureName = + "tweet.mediaunderstanding.tweet_annotations.sensitive_category_probabilities" + + def updateMediaCategoryStats( + candidates: Seq[CandidateDetails[PushCandidate]] + )( + implicit statsReceiver: StatsReceiver + ) = { + + val statScope = statsReceiver.scope("mediaStats") + val filteredCandidates = candidates.filter { candidate => + !candidate.candidate.sparseContinuousFeatures + .getOrElse(sensitiveMediaCategoryFeatureName, Map.empty[String, Double]).contains( + nudityCategoryId) + } + + if (filteredCandidates.isEmpty) + statScope.counter("emptyCandidateListAfterNudityFilter").incr() + else + statScope.counter("nonEmptyCandidateListAfterNudityFilter").incr() + candidates.foreach { candidate => + statScope.counter("totalCandidates").incr() + val mediaFeature = candidate.candidate.sparseContinuousFeatures + .getOrElse(sensitiveMediaCategoryFeatureName, Map.empty[String, Double]) + if (mediaFeature.nonEmpty) { + val mediaCategoryByMaxScore = mediaFeature.maxBy(_._2)._1 + statScope + .scope("mediaCategoryByMaxScore").counter(mediaIdToCategoryMapping + .getOrElse(mediaCategoryByMaxScore, "undefined")).incr() + + mediaFeature.keys.map { feature => + statScope + .scope("mediaCategory").counter(mediaIdToCategoryMapping + .getOrElse(feature, "undefined")).incr() + } + } + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.scala new file mode 100644 index 0000000000..e96ecb8176 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.scala @@ -0,0 +1,187 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.util.TimeUtil +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} +import com.twitter.util.Future +import com.twitter.util.Time +import java.util.Calendar +import java.util.TimeZone + +case class MinDurationModifierCalculator() { + + private def mapCountryCodeToTimeZone( + countryCode: String, + stats: StatsReceiver + ): Option[Calendar] = { + PushConstants.countryCodeToTimeZoneMap + .get(countryCode.toUpperCase).map(timezone => + Calendar.getInstance(TimeZone.getTimeZone(timezone))) + } + + private def transformToHour( + dayOfHour: Int + ): Int = { + if (dayOfHour < 0) dayOfHour + 24 + else dayOfHour + } + + private def getMinDurationByHourOfDay( + hourOfDay: Int, + startTimeList: Seq[Int], + endTimeList: Seq[Int], + minDurationTimeModifierConst: Seq[Int], + stats: StatsReceiver + ): Option[Int] = { + val scopedStats = stats.scope("getMinDurationByHourOfDay") + scopedStats.counter("request").incr() + val durationOpt = (startTimeList, endTimeList, minDurationTimeModifierConst).zipped.toList + .filter { + case (startTime, endTime, _) => + if (startTime <= endTime) hourOfDay >= startTime && hourOfDay < endTime + else (hourOfDay >= startTime) || hourOfDay < endTime + case _ => false + }.map { + case (_, _, modifier) => modifier + }.headOption + durationOpt match { + case Some(duration) => scopedStats.counter(s"$duration.minutes").incr() + case _ => scopedStats.counter("none").incr() + } + durationOpt + } + + def getMinDurationModifier( + target: Target, + calendar: Calendar, + stats: StatsReceiver + ): Option[Int] = { + val startTimeList = target.params(FSParams.MinDurationModifierStartHourList) + val endTimeList = target.params(FSParams.MinDurationModifierEndHourList) + val minDurationTimeModifierConst = target.params(FSParams.MinDurationTimeModifierConst) + if (startTimeList.length != endTimeList.length || minDurationTimeModifierConst.length != startTimeList.length) { + None + } else { + val hourOfDay = calendar.get(Calendar.HOUR_OF_DAY) + getMinDurationByHourOfDay( + hourOfDay, + startTimeList, + endTimeList, + minDurationTimeModifierConst, + stats) + } + } + + def getMinDurationModifier( + target: Target, + countryCodeOpt: Option[String], + stats: StatsReceiver + ): Option[Int] = { + val scopedStats = stats + .scope("getMinDurationModifier") + scopedStats.counter("total_requests").incr() + + countryCodeOpt match { + case Some(countryCode) => + scopedStats + .counter("country_code_exists").incr() + val calendarOpt = mapCountryCodeToTimeZone(countryCode, scopedStats) + calendarOpt.flatMap(calendar => getMinDurationModifier(target, calendar, scopedStats)) + case _ => None + } + } + + def getMinDurationModifier(target: Target, stats: StatsReceiver): Future[Option[Int]] = { + val scopedStats = stats + .scope("getMinDurationModifier") + scopedStats.counter("total_requests").incr() + + val startTimeList = target.params(FSParams.MinDurationModifierStartHourList) + val endTimeList = target.params(FSParams.MinDurationModifierEndHourList) + val minDurationTimeModifierConst = target.params(FSParams.MinDurationTimeModifierConst) + if (startTimeList.length != endTimeList.length || minDurationTimeModifierConst.length != startTimeList.length) { + Future.value(None) + } else { + target.localTimeInHHMM.map { + case (hourOfDay, _) => + getMinDurationByHourOfDay( + hourOfDay, + startTimeList, + endTimeList, + minDurationTimeModifierConst, + scopedStats) + case _ => None + } + } + } + + def getMinDurationModifierByUserOpenedHistory( + target: Target, + openedPushByHourAggregatedOpt: Option[Map[Int, Int]], + stats: StatsReceiver + ): Option[Int] = { + val scopedStats = stats + .scope("getMinDurationModifierByUserOpenedHistory") + scopedStats.counter("total_requests").incr() + openedPushByHourAggregatedOpt match { + case Some(openedPushByHourAggregated) => + if (openedPushByHourAggregated.isEmpty) { + scopedStats.counter("openedPushByHourAggregated_empty").incr() + None + } else { + val currentUTCHour = TimeUtil.hourOfDay(Time.now) + val utcHourWithMaxOpened = if (target.params(FSParams.EnableRandomHourForQuickSend)) { + (target.targetId % 24).toInt + } else { + openedPushByHourAggregated.maxBy(_._2)._1 + } + val numOfMaxOpened = openedPushByHourAggregated.maxBy(_._2)._2 + if (numOfMaxOpened >= target.params(FSParams.SendTimeByUserHistoryMaxOpenedThreshold)) { + scopedStats.counter("pass_experiment_bucket_threshold").incr() + if (numOfMaxOpened >= target + .params(FSParams.SendTimeByUserHistoryMaxOpenedThreshold)) { // only update if number of opened pushes meet threshold + scopedStats.counter("pass_max_threshold").incr() + val quickSendBeforeHours = + target.params(FSParams.SendTimeByUserHistoryQuickSendBeforeHours) + val quickSendAfterHours = + target.params(FSParams.SendTimeByUserHistoryQuickSendAfterHours) + + val hoursToLessSend = target.params(FSParams.SendTimeByUserHistoryNoSendsHours) + + val quickSendTimeMinDurationInMinute = + target.params(FSParams.SendTimeByUserHistoryQuickSendMinDurationInMinute) + val noSendTimeMinDuration = + target.params(FSParams.SendTimeByUserHistoryNoSendMinDuration) + + val startTimeForNoSend = transformToHour( + utcHourWithMaxOpened - quickSendBeforeHours - hoursToLessSend) + val startTimeForQuickSend = transformToHour( + utcHourWithMaxOpened - quickSendBeforeHours) + val endTimeForNoSend = + transformToHour(utcHourWithMaxOpened - quickSendBeforeHours) + val endTimeForQuickSend = + transformToHour(utcHourWithMaxOpened + quickSendAfterHours) + 1 + + val startTimeList = Seq(startTimeForNoSend, startTimeForQuickSend) + val endTimeList = Seq(endTimeForNoSend, endTimeForQuickSend) + val minDurationTimeModifierConst = + Seq(noSendTimeMinDuration, quickSendTimeMinDurationInMinute) + + getMinDurationByHourOfDay( + currentUTCHour, + startTimeList, + endTimeList, + minDurationTimeModifierConst, + scopedStats) + + } else None + } else None + } + case _ => + None + } + } + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.scala new file mode 100644 index 0000000000..33333c4c46 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.scala @@ -0,0 +1,16 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TargetUser + +object MrUserStateUtil { + def updateMrUserStateStats(target: TargetUser)(implicit statsReceiver: StatsReceiver) = { + statsReceiver.counter("AllUserStates").incr() + target.targetMrUserState.map { + case Some(state) => + statsReceiver.counter(state.name).incr() + case _ => + statsReceiver.counter("UnknownUserState").incr() + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.scala new file mode 100644 index 0000000000..01c8d0a725 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.scala @@ -0,0 +1,126 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation + +object NsfwPersonalizationUtil { + def computeNsfwUserStats( + targetNsfwInfo: Option[NsfwInfo] + )( + implicit statsReceiver: StatsReceiver + ): Unit = { + + def computeNsfwProfileVisitStats(sReceiver: StatsReceiver, nsfwProfileVisits: Long): Unit = { + if (nsfwProfileVisits >= 1) + sReceiver.counter("nsfwProfileVisits_gt_1").incr() + if (nsfwProfileVisits >= 2) + sReceiver.counter("nsfwProfileVisits_gt_2").incr() + if (nsfwProfileVisits >= 3) + sReceiver.counter("nsfwProfileVisits_gt_3").incr() + if (nsfwProfileVisits >= 5) + sReceiver.counter("nsfwProfileVisits_gt_5").incr() + if (nsfwProfileVisits >= 8) + sReceiver.counter("nsfwProfileVisits_gt_8").incr() + } + + def computeRatioStats( + sReceiver: StatsReceiver, + ratio: Int, + statName: String, + intervals: List[Double] = List(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9) + ): Unit = { + intervals.foreach { i => + if (ratio > i * 10000) + sReceiver.counter(f"${statName}_greater_than_${i}").incr() + } + } + val sReceiver = statsReceiver.scope("nsfw_personalization") + sReceiver.counter("AllUsers").incr() + + (targetNsfwInfo) match { + case (Some(nsfwInfo)) => + val sensitive = nsfwInfo.senstiveOptIn.getOrElse(false) + val nsfwFollowRatio = + nsfwInfo.nsfwFollowRatio + val totalFollows = nsfwInfo.totalFollowCount + val numNsfwProfileVisits = nsfwInfo.nsfwProfileVisits + val nsfwRealGraphScore = nsfwInfo.realGraphScore + val nsfwSearchScore = nsfwInfo.searchNsfwScore + val totalSearches = nsfwInfo.totalSearches + val realGraphScore = nsfwInfo.realGraphScore + val searchScore = nsfwInfo.searchNsfwScore + + if (sensitive) + sReceiver.counter("sensitiveOptInEnabled").incr() + else + sReceiver.counter("sensitiveOptInDisabled").incr() + + computeRatioStats(sReceiver, nsfwFollowRatio, "nsfwRatio") + computeNsfwProfileVisitStats(sReceiver, numNsfwProfileVisits) + computeRatioStats(sReceiver, nsfwRealGraphScore.toInt, "nsfwRealGraphScore") + + if (totalSearches >= 10) + computeRatioStats(sReceiver, nsfwSearchScore.toInt, "nsfwSearchScore") + if (searchScore == 0) + sReceiver.counter("lowSearchScore").incr() + if (realGraphScore < 500) + sReceiver.counter("lowRealScore").incr() + if (numNsfwProfileVisits == 0) + sReceiver.counter("lowProfileVisit").incr() + if (nsfwFollowRatio == 0) + sReceiver.counter("lowFollowScore").incr() + + if (totalSearches > 10 && searchScore > 5000) + sReceiver.counter("highSearchScore").incr() + if (realGraphScore > 7000) + sReceiver.counter("highRealScore").incr() + if (numNsfwProfileVisits > 5) + sReceiver.counter("highProfileVisit").incr() + if (totalFollows > 10 && nsfwFollowRatio > 7000) + sReceiver.counter("highFollowScore").incr() + + if (searchScore == 0 && realGraphScore <= 500 && numNsfwProfileVisits == 0 && nsfwFollowRatio == 0) + sReceiver.counter("lowIntent").incr() + if ((totalSearches > 10 && searchScore > 5000) || realGraphScore > 7000 || numNsfwProfileVisits > 5 || (totalFollows > 10 && nsfwFollowRatio > 7000)) + sReceiver.counter("highIntent").incr() + case _ => + } + } +} + +case class NsfwInfo(nsfwUserSegmentation: NSFWUserSegmentation) { + + val scalingFactor = 10000 // to convert float to int as custom fields cannot be float + val senstiveOptIn: Option[Boolean] = nsfwUserSegmentation.nsfwView + val totalFollowCount: Long = nsfwUserSegmentation.totalFollowCnt.getOrElse(0L) + val nsfwFollowCnt: Long = + nsfwUserSegmentation.nsfwAdminOrHighprecOrAgathaGtP98FollowsCnt.getOrElse(0L) + val nsfwFollowRatio: Int = { + if (totalFollowCount != 0) { + (nsfwFollowCnt * scalingFactor / totalFollowCount).toInt + } else 0 + } + val nsfwProfileVisits: Long = + nsfwUserSegmentation.nsfwAdminOrHighPrecOrAgathaGtP98Visits + .map(_.numProfilesInLast14Days).getOrElse(0L) + val realGraphScore: Int = + nsfwUserSegmentation.realGraphMetrics + .map { rm => + if (rm.totalOutboundRGScore != 0) + rm.totalNsfwAdmHPAgthGtP98OutboundRGScore * scalingFactor / rm.totalOutboundRGScore + else 0d + }.getOrElse(0d).toInt + val totalSearches: Long = + nsfwUserSegmentation.searchMetrics.map(_.numNonTrndSrchInLast14Days).getOrElse(0L) + val searchNsfwScore: Int = nsfwUserSegmentation.searchMetrics + .map { sm => + if (sm.numNonTrndNonHshtgSrchInLast14Days != 0) + sm.numNonTrndNonHshtgGlobalNsfwSrchInLast14Days.toDouble * scalingFactor / sm.numNonTrndNonHshtgSrchInLast14Days + else 0 + }.getOrElse(0d).toInt + val hasReported: Boolean = + nsfwUserSegmentation.notifFeedbackMetrics.exists(_.notifReportMetrics.exists(_.countTotal != 0)) + val hasDisliked: Boolean = + nsfwUserSegmentation.notifFeedbackMetrics + .exists(_.notifDislikeMetrics.exists(_.countTotal != 0)) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.scala new file mode 100644 index 0000000000..ac4aba8a77 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.scala @@ -0,0 +1,230 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.MagicFanoutEventCandidate +import com.twitter.frigate.common.history.History +import com.twitter.frigate.common.rec_types.RecTypes +import com.twitter.frigate.common.store.deviceinfo.DeviceInfo +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes +import com.twitter.frigate.pushservice.model.ibis.PushOverrideInfo +import com.twitter.frigate.pushservice.params.PushConstants +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} +import com.twitter.frigate.thriftscala.CollapseInfo +import com.twitter.frigate.thriftscala.CommonRecommendationType +import com.twitter.frigate.thriftscala.CommonRecommendationType.MagicFanoutSportsEvent +import com.twitter.frigate.thriftscala.OverrideInfo +import com.twitter.util.Future +import java.util.UUID + +object OverrideNotificationUtil { + + /** + * Gets Override Info for the current notification. + * @param candidate [[PushCandidate]] object representing the recommendation candidate + * @param stats StatsReceiver to track stats for this function as well as the subsequent funcs. called + * @return Returns OverrideInfo if CollapseInfo exists, else None + */ + + def getOverrideInfo( + candidate: PushCandidate, + stats: StatsReceiver + ): Future[Option[OverrideInfo]] = { + if (candidate.target.isLoggedOutUser) { + Future.None + } else if (isOverrideEnabledForCandidate(candidate)) + getCollapseInfo(candidate, stats).map(_.map(OverrideInfo(_))) + else Future.None + } + + private def getCollapseInfo( + candidate: PushCandidate, + stats: StatsReceiver + ): Future[Option[CollapseInfo]] = { + val target = candidate.target + for { + targetHistory <- target.history + deviceInfo <- target.deviceInfo + } yield getCollapseInfo(target, targetHistory, deviceInfo, stats) + } + + /** + * Get Collapse Info for the current notification. + * @param target Push Target - recipient of the notification + * @param targetHistory Target's History + * @param deviceInfoOpt `Option` of the Target's Device Info + * @param stats StatsReceiver to track stats for this function as well as the subsequent funcs. called + * @return Returns CollapseInfo if the Target is eligible for Override Notifs, else None + */ + def getCollapseInfo( + target: PushTypes.Target, + targetHistory: History, + deviceInfoOpt: Option[DeviceInfo], + stats: StatsReceiver + ): Option[CollapseInfo] = { + val overrideInfoOfLastNotif = + PushOverrideInfo.getOverrideInfoOfLastEligiblePushNotif( + targetHistory, + target.params(FSParams.OverrideNotificationsLookbackDurationForOverrideInfo), + stats) + overrideInfoOfLastNotif match { + case Some(prevOverrideInfo) if isOverrideEnabled(target, deviceInfoOpt, stats) => + val notifsInLastOverrideChain = + PushOverrideInfo.getMrPushNotificationsInOverrideChain( + targetHistory, + prevOverrideInfo.collapseInfo.overrideChainId, + stats) + val numNotifsInLastOverrideChain = notifsInLastOverrideChain.size + val timestampOfFirstNotifInOverrideChain = + PushOverrideInfo + .getTimestampInMillisForFrigateNotification( + notifsInLastOverrideChain.last, + targetHistory, + stats).getOrElse(PushConstants.DefaultLookBackForHistory.ago.inMilliseconds) + if (numNotifsInLastOverrideChain < target.params(FSParams.MaxMrPushSends24HoursParam) && + timestampOfFirstNotifInOverrideChain > PushConstants.DefaultLookBackForHistory.ago.inMilliseconds) { + Some(prevOverrideInfo.collapseInfo) + } else { + val prevCollapseId = prevOverrideInfo.collapseInfo.collapseId + val newOverrideChainId = UUID.randomUUID.toString.replaceAll("-", "") + Some(CollapseInfo(prevCollapseId, newOverrideChainId)) + } + case None if isOverrideEnabled(target, deviceInfoOpt, stats) => + val newOverrideChainId = UUID.randomUUID.toString.replaceAll("-", "") + Some(CollapseInfo("", newOverrideChainId)) + case _ => None // Override is disabled for everything else + } + } + + /** + * Gets the collapse and impression identifier for the current override notification + * @param target Push Target - recipient of the notification + * @param stats StatsReceiver to track stats for this function as well as the subsequent funcs. called + * @return A Future of Collapse ID as well as the Impression ID. + */ + def getCollapseAndImpressionIdForOverride( + candidate: PushCandidate + ): Future[Option[(String, Seq[String])]] = { + if (isOverrideEnabledForCandidate(candidate)) { + val target = candidate.target + val stats = candidate.statsReceiver + Future.join(target.history, target.deviceInfo).map { + case (targetHistory, deviceInfoOpt) => + val collapseInfoOpt = getCollapseInfo(target, targetHistory, deviceInfoOpt, stats) + + val impressionIds = candidate.commonRecType match { + case MagicFanoutSportsEvent + if target.params(FSParams.EnableEventIdBasedOverrideForSportsCandidates) => + PushOverrideInfo.getImpressionIdsForPrevEligibleMagicFanoutEventCandidates( + targetHistory, + target.params(FSParams.OverrideNotificationsLookbackDurationForImpressionId), + stats, + MagicFanoutSportsEvent, + candidate + .asInstanceOf[RawCandidate with MagicFanoutEventCandidate].eventId + ) + case _ => + PushOverrideInfo.getImpressionIdsOfPrevEligiblePushNotif( + targetHistory, + target.params(FSParams.OverrideNotificationsLookbackDurationForImpressionId), + stats) + } + + collapseInfoOpt match { + case Some(collapseInfo) if impressionIds.nonEmpty => + val notifsInLastOverrideChain = + PushOverrideInfo.getMrPushNotificationsInOverrideChain( + targetHistory, + collapseInfo.overrideChainId, + stats) + stats + .scope("OverrideNotificationUtil").stat("number_of_notifications_sent").add( + notifsInLastOverrideChain.size + 1) + Some((collapseInfo.collapseId, impressionIds)) + case _ => None + } + case _ => None + } + } else Future.None + } + + /** + * Checks to see if override notifications are enabled based on the Target's Device Info and Params + * @param target Push Target - recipient of the notification + * @param deviceInfoOpt `Option` of the Target's Device Info + * @param stats StatsReceiver to track stats for this function + * @return Returns True if Override Notifications are enabled for the provided + * Target, else False. + */ + private def isOverrideEnabled( + target: PushTypes.Target, + deviceInfoOpt: Option[DeviceInfo], + stats: StatsReceiver + ): Boolean = { + val scopedStats = stats.scope("OverrideNotificationUtil").scope("isOverrideEnabled") + val enabledForAndroidCounter = scopedStats.counter("android_enabled") + val disabledForAndroidCounter = scopedStats.counter("android_disabled") + val enabledForIosCounter = scopedStats.counter("ios_enabled") + val disabledForIosCounter = scopedStats.counter("ios_disabled") + val disabledForOtherDevicesCounter = scopedStats.counter("other_disabled") + + val isPrimaryDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) + val isPrimaryDeviceIos = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) + + lazy val validAndroidDevice = + isPrimaryDeviceAndroid && target.params(FSParams.EnableOverrideNotificationsForAndroid) + lazy val validIosDevice = + isPrimaryDeviceIos && target.params(FSParams.EnableOverrideNotificationsForIos) + + if (isPrimaryDeviceAndroid) { + if (validAndroidDevice) enabledForAndroidCounter.incr() else disabledForAndroidCounter.incr() + } else if (isPrimaryDeviceIos) { + if (validIosDevice) enabledForIosCounter.incr() else disabledForIosCounter.incr() + } else { + disabledForOtherDevicesCounter.incr() + } + + validAndroidDevice || validIosDevice + } + + /** + * Checks if override is enabled for the currently supported types for SendHandler or not. + * This method is package private for unit testing. + * @param candidate [[PushCandidate]] + * @param stats StatsReceiver to track statistics for this function + * @return Returns True if override notifications are enabled for the current type, otherwise False. + */ + private def isOverrideEnabledForSendHandlerCandidate( + candidate: PushCandidate + ): Boolean = { + val scopedStats = candidate.statsReceiver + .scope("OverrideNotificationUtil").scope("isOverrideEnabledForSendHandlerType") + + val overrideSupportedTypesForSpaces: Set[CommonRecommendationType] = Set( + CommonRecommendationType.SpaceSpeaker, + CommonRecommendationType.SpaceHost + ) + + val isOverrideSupportedForSpaces = { + overrideSupportedTypesForSpaces.contains(candidate.commonRecType) && + candidate.target.params(FSParams.EnableOverrideForSpaces) + } + + val isOverrideSupportedForSports = { + candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent && + candidate.target + .params(PushFeatureSwitchParams.EnableOverrideForSportsCandidates) + } + + val isOverrideSupported = isOverrideSupportedForSpaces || isOverrideSupportedForSports + + scopedStats.counter(s"$isOverrideSupported").incr() + isOverrideSupported + } + + private[util] def isOverrideEnabledForCandidate(candidate: PushCandidate) = + !RecTypes.isSendHandlerType( + candidate.commonRecType) || isOverrideEnabledForSendHandlerCandidate(candidate) +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.scala new file mode 100644 index 0000000000..7bc29cf01e --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.scala @@ -0,0 +1,151 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.contentrecommender.thriftscala.MetricTag +import com.twitter.frigate.common.base.AlgorithmScore +import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate +import com.twitter.frigate.common.base.SocialContextAction +import com.twitter.frigate.common.base.TopicCandidate +import com.twitter.frigate.common.base.TripCandidate +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.thriftscala.{SocialContextAction => TSocialContextAction} +import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} +import com.twitter.frigate.thriftscala._ +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.topiclisting.utt.LocalizedEntity +import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain +import scala.collection.Seq + +case class MediaCRT( + crt: CRT, + photoCRT: CRT, + videoCRT: CRT) + +object PushAdaptorUtil { + + def getFrigateNotificationForUser( + crt: CRT, + userId: Long, + scActions: Seq[SocialContextAction], + pushCopyId: Option[Int], + ntabCopyId: Option[Int] + ): FrigateNotification = { + + val thriftSCActions = scActions.map { scAction => + TSocialContextAction( + scAction.userId, + scAction.timestampInMillis, + scAction.tweetId + ) + } + FrigateNotification( + crt, + NotificationDisplayLocation.PushToMobileDevice, + userNotification = Some(UserNotification(userId, thriftSCActions)), + pushCopyId = pushCopyId, + ntabCopyId = ntabCopyId + ) + } + + def getFrigateNotificationForTweet( + crt: CRT, + tweetId: Long, + scActions: Seq[TSocialContextAction], + authorIdOpt: Option[Long], + pushCopyId: Option[Int], + ntabCopyId: Option[Int], + simclusterId: Option[Int], + semanticCoreEntityIds: Option[List[Long]], + candidateContent: Option[CandidateContent], + trendId: Option[String], + tweetTripDomain: Option[scala.collection.Set[TripDomain]] = None + ): FrigateNotification = { + FrigateNotification( + crt, + NotificationDisplayLocation.PushToMobileDevice, + tweetNotification = Some( + TweetNotification( + tweetId, + scActions, + authorIdOpt, + simclusterId, + semanticCoreEntityIds, + trendId, + tripDomain = tweetTripDomain) + ), + pushCopyId = pushCopyId, + ntabCopyId = ntabCopyId, + candidateContent = candidateContent + ) + } + + def getFrigateNotificationForTweetWithSocialContextActions( + crt: CRT, + tweetId: Long, + scActions: Seq[SocialContextAction], + authorIdOpt: Option[Long], + pushCopyId: Option[Int], + ntabCopyId: Option[Int], + candidateContent: Option[CandidateContent], + semanticCoreEntityIds: Option[List[Long]], + trendId: Option[String] + ): FrigateNotification = { + + val thriftSCActions = scActions.map { scAction => + TSocialContextAction( + scAction.userId, + scAction.timestampInMillis, + scAction.tweetId + ) + } + + getFrigateNotificationForTweet( + crt = crt, + tweetId = tweetId, + scActions = thriftSCActions, + authorIdOpt = authorIdOpt, + pushCopyId = pushCopyId, + ntabCopyId = ntabCopyId, + simclusterId = None, + candidateContent = candidateContent, + semanticCoreEntityIds = semanticCoreEntityIds, + trendId = trendId + ) + } + + def generateOutOfNetworkTweetCandidates( + inputTarget: Target, + id: Long, + mediaCRT: MediaCRT, + result: Option[TweetyPieResult], + localizedEntity: Option[LocalizedEntity] = None, + isMrBackfillFromCR: Option[Boolean] = None, + tagsFromCR: Option[Seq[MetricTag]] = None, + score: Option[Double] = None, + algorithmTypeCR: Option[String] = None, + tripTweetDomain: Option[scala.collection.Set[TripDomain]] = None + ): RawCandidate + with OutOfNetworkTweetCandidate + with TopicCandidate + with TripCandidate + with AlgorithmScore = { + new RawCandidate + with OutOfNetworkTweetCandidate + with TopicCandidate + with TripCandidate + with AlgorithmScore { + override val tweetId: Long = id + override val target: Target = inputTarget + override val tweetyPieResult: Option[TweetyPieResult] = result + override val localizedUttEntity: Option[LocalizedEntity] = localizedEntity + override val semanticCoreEntityId: Option[Long] = localizedEntity.map(_.entityId) + override def commonRecType: CRT = + getMediaBasedCRT(mediaCRT.crt, mediaCRT.photoCRT, mediaCRT.videoCRT) + override def isMrBackfillCR: Option[Boolean] = isMrBackfillFromCR + override def tagsCR: Option[Seq[MetricTag]] = tagsFromCR + override def algorithmScore: Option[Double] = score + override def algorithmCR: Option[String] = algorithmTypeCR + override def tripDomain: Option[collection.Set[TripDomain]] = tripTweetDomain + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.scala new file mode 100644 index 0000000000..0afa90fed9 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.scala @@ -0,0 +1,49 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.frigate.common.store.deviceinfo.DeviceInfo +import com.twitter.onboarding.task.service.models.external.PermissionState +import com.twitter.permissions_storage.thriftscala.AppPermission +import com.twitter.storehaus.ReadableStore +import com.twitter.util.Future + +object PushAppPermissionUtil { + + final val AddressBookPermissionKey = "addressBook" + final val SyncStateKey = "syncState" + final val SyncStateOnValue = "on" + + /** + * Obtains the specified target's App Permissions, based on their primary device. + * @param targetId Target's Identifier + * @param permissionName The permission type we are querying for (address book, geolocation, etc.) + * @param deviceInfoFut Device info of the Target, presented as a Future + * @param appPermissionStore Readable Store which allows us to query the App Permission Strato Column + * @return Returns the AppPermission of the Target, presented as a Future + */ + def getAppPermission( + targetId: Long, + permissionName: String, + deviceInfoFut: Future[Option[DeviceInfo]], + appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission] + ): Future[Option[AppPermission]] = { + deviceInfoFut.flatMap { deviceInfoOpt => + val primaryDeviceIdOpt = deviceInfoOpt.flatMap(_.primaryDeviceId) + primaryDeviceIdOpt match { + case Some(primaryDeviceId) => + val queryKey = (targetId, (primaryDeviceId, permissionName)) + appPermissionStore.get(queryKey) + case _ => Future.None + } + } + } + + def hasTargetUploadedAddressBook( + appPermissionOpt: Option[AppPermission] + ): Boolean = { + appPermissionOpt.exists { appPermission => + val syncState = appPermission.metadata.get(SyncStateKey) + appPermission.systemPermissionState == PermissionState.On && syncState + .exists(_.equalsIgnoreCase(SyncStateOnValue)) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.scala new file mode 100644 index 0000000000..d5d79c4fde --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.scala @@ -0,0 +1,184 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.TargetUser +import com.twitter.frigate.common.candidate.FrigateHistory +import com.twitter.frigate.common.candidate.ResurrectedUserDetails +import com.twitter.frigate.common.candidate.TargetABDecider +import com.twitter.frigate.common.candidate.UserDetails +import com.twitter.frigate.pushcap.thriftscala.ModelType +import com.twitter.frigate.pushcap.thriftscala.PushcapInfo +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +import com.twitter.frigate.scribe.thriftscala.PushCapInfo +import com.twitter.util.Duration +import com.twitter.util.Future + +case class PushCapFatigueInfo( + pushcap: Int, + fatigueInterval: Duration) {} + +object PushCapUtil { + + def getDefaultPushCap(target: Target): Future[Int] = { + Future.value(target.params(PushFeatureSwitchParams.MaxMrPushSends24HoursParam)) + } + + def getMinimumRestrictedPushcapInfo( + restrictedPushcap: Int, + originalPushcapInfo: PushcapInfo, + statsReceiver: StatsReceiver + ): PushcapInfo = { + if (originalPushcapInfo.pushcap < restrictedPushcap) { + statsReceiver + .scope("minModelPushcapRestrictions").counter( + f"num_users_adjusted_from_${originalPushcapInfo.pushcap}_to_${restrictedPushcap}").incr() + PushcapInfo( + pushcap = restrictedPushcap.toShort, + modelType = ModelType.NoModel, + timestamp = 0L, + fatigueMinutes = Some((24L / restrictedPushcap) * 60L) + ) + } else originalPushcapInfo + } + + def getPushCapFatigue( + target: Target, + statsReceiver: StatsReceiver + ): Future[PushCapFatigueInfo] = { + val pushCapStats = statsReceiver.scope("pushcap_stats") + target.dynamicPushcap + .map { dynamicPushcapOpt => + val pushCap: Int = dynamicPushcapOpt match { + case Some(pushcapInfo) => pushcapInfo.pushcap + case _ => target.params(PushFeatureSwitchParams.MaxMrPushSends24HoursParam) + } + + pushCapStats.stat("pushCapValueStats").add(pushCap) + pushCapStats + .scope("pushCapValueCount").counter(f"num_users_with_pushcap_$pushCap").incr() + + target.finalPushcapAndFatigue += "pushPushCap" -> PushCapInfo("pushPushCap", pushCap.toByte) + + PushCapFatigueInfo(pushCap, 24.hours) + } + } + + def getMinDurationsSincePushWithoutUsingPushCap( + target: TargetUser + with TargetABDecider + with FrigateHistory + with UserDetails + with ResurrectedUserDetails + )( + implicit statsReceiver: StatsReceiver + ): Duration = { + val minDurationSincePush = + if (target.params(PushFeatureSwitchParams.EnableGraduallyRampUpNotification)) { + val daysInterval = + target.params(PushFeatureSwitchParams.GraduallyRampUpPhaseDurationDays).inDays.toDouble + val daysSinceActivation = + if (target.isResurrectedUser && target.timeSinceResurrection.isDefined) { + target.timeSinceResurrection.map(_.inDays.toDouble).get + } else { + target.timeElapsedAfterSignup.inDays.toDouble + } + val phaseInterval = + Math.max( + 1, + Math.ceil(daysSinceActivation / daysInterval).toInt + ) + val minDuration = 24 / phaseInterval + val finalMinDuration = + Math.max(4, minDuration).hours + statsReceiver + .scope("GraduallyRampUpFinalMinDuration").counter(s"$finalMinDuration.hours").incr() + finalMinDuration + } else { + target.params(PushFeatureSwitchParams.MinDurationSincePushParam) + } + statsReceiver + .scope("minDurationsSincePushWithoutUsingPushCap").counter( + s"$minDurationSincePush.hours").incr() + minDurationSincePush + } + + def getMinDurationSincePush( + target: Target, + statsReceiver: StatsReceiver + ): Future[Duration] = { + val minDurationStats: StatsReceiver = statsReceiver.scope("pushcapMinDuration_stats") + val minDurationModifierCalculator = + MinDurationModifierCalculator() + val openedPushByHourAggregatedFut = + if (target.params(PushFeatureSwitchParams.EnableQueryUserOpenedHistory)) + target.openedPushByHourAggregated + else Future.None + Future + .join( + target.dynamicPushcap, + target.accountCountryCode, + openedPushByHourAggregatedFut + ) + .map { + case (dynamicPushcapOpt, countryCodeOpt, openedPushByHourAggregated) => + val minDurationSincePush: Duration = { + val isGraduallyRampingUpResurrected = target.isResurrectedUser && target.params( + PushFeatureSwitchParams.EnableGraduallyRampUpNotification) + if (isGraduallyRampingUpResurrected || target.params( + PushFeatureSwitchParams.EnableExplicitPushCap)) { + getMinDurationsSincePushWithoutUsingPushCap(target)(minDurationStats) + } else { + dynamicPushcapOpt match { + case Some(pushcapInfo) => + pushcapInfo.fatigueMinutes match { + case Some(fatigueMinutes) => (fatigueMinutes / 60).hours + case _ if pushcapInfo.pushcap > 0 => (24 / pushcapInfo.pushcap).hours + case _ => getMinDurationsSincePushWithoutUsingPushCap(target)(minDurationStats) + } + case _ => + getMinDurationsSincePushWithoutUsingPushCap(target)(minDurationStats) + } + } + } + + val modifiedMinDurationSincePush = + if (target.params(PushFeatureSwitchParams.EnableMinDurationModifier)) { + val modifierHourOpt = + minDurationModifierCalculator.getMinDurationModifier( + target, + countryCodeOpt, + statsReceiver.scope("MinDuration")) + modifierHourOpt match { + case Some(modifierHour) => modifierHour.hours + case _ => minDurationSincePush + } + } else if (target.params( + PushFeatureSwitchParams.EnableMinDurationModifierByUserHistory)) { + val modifierMinuteOpt = + minDurationModifierCalculator.getMinDurationModifierByUserOpenedHistory( + target, + openedPushByHourAggregated, + statsReceiver.scope("MinDuration")) + + modifierMinuteOpt match { + case Some(modifierMinute) => modifierMinute.minutes + case _ => minDurationSincePush + } + } else minDurationSincePush + + target.finalPushcapAndFatigue += "pushFatigue" -> PushCapInfo( + "pushFatigue", + modifiedMinDurationSincePush.inHours.toByte) + + minDurationStats + .stat("minDurationSincePushValueStats").add(modifiedMinDurationSincePush.inHours) + minDurationStats + .scope("minDurationSincePushValueCount").counter( + s"$modifiedMinDurationSincePush").incr() + + modifiedMinDurationSincePush + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.scala new file mode 100644 index 0000000000..d191d742af --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.scala @@ -0,0 +1,57 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.frigate.common.store.deviceinfo.DeviceInfo +import com.twitter.frigate.common.store.deviceinfo.MobileClientType +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.util.Future +import com.twitter.finagle.stats.NullStatsReceiver +import com.twitter.finagle.stats.StatsReceiver + +object PushDeviceUtil { + + def isPrimaryDeviceAndroid(deviceInfoOpt: Option[DeviceInfo]): Boolean = { + deviceInfoOpt.exists { + _.guessedPrimaryClient.exists { clientType => + (clientType == MobileClientType.Android) || (clientType == MobileClientType.AndroidLite) + } + } + } + + def isPrimaryDeviceIOS(deviceInfoOpt: Option[DeviceInfo]): Boolean = { + deviceInfoOpt.exists { + _.guessedPrimaryClient.exists { clientType => + (clientType == MobileClientType.Iphone) || (clientType == MobileClientType.Ipad) + } + } + } + + def isPushRecommendationsEligible(target: Target): Future[Boolean] = + target.deviceInfo.map(_.exists(_.isRecommendationsEligible)) + + def isTopicsEligible( + target: Target, + statsReceiver: StatsReceiver = NullStatsReceiver + ): Future[Boolean] = { + val isTopicsSkipFatigue = Future.True + + Future.join(isTopicsSkipFatigue, target.deviceInfo.map(_.exists(_.isTopicsEligible))).map { + case (isTopicsNotFatigue, isTopicsEligibleSetting) => + isTopicsNotFatigue && isTopicsEligibleSetting + } + } + + def isSpacesEligible(target: Target): Future[Boolean] = + target.deviceInfo.map(_.exists(_.isSpacesEligible)) + + def isNtabOnlyEligible: Future[Boolean] = { + Future.False + } + + def isRecommendationsEligible(target: Target): Future[Boolean] = { + Future.join(isPushRecommendationsEligible(target), isNtabOnlyEligible).map { + case (isPushRecommendation, isNtabOnly) => isPushRecommendation || isNtabOnly + case _ => false + } + } + +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.scala new file mode 100644 index 0000000000..7567726bf4 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.scala @@ -0,0 +1,36 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.util.Future + +object PushIbisUtil { + + def getSocialContextModelValues(socialContextUserIds: Seq[Long]): Map[String, String] = { + + val socialContextSize = socialContextUserIds.size + + val (displaySocialContexts, otherCount) = { + if (socialContextSize < 3) (socialContextUserIds, 0) + else (socialContextUserIds.take(1), socialContextSize - 1) + } + + val usersValue = displaySocialContexts.map(_.toString).mkString(",") + + if (otherCount > 0) Map("social_users" -> s"$usersValue+$otherCount") + else Map("social_users" -> usersValue) + } + + def mergeFutModelValues( + mvFut1: Future[Map[String, String]], + mvFut2: Future[Map[String, String]] + ): Future[Map[String, String]] = { + Future.join(mvFut1, mvFut2).map { + case (mv1, mv2) => mv1 ++ mv2 + } + } + + def mergeModelValues( + mvFut1: Future[Map[String, String]], + mv2: Map[String, String] + ): Future[Map[String, String]] = + mvFut1.map { mv1 => mv1 ++ mv2 } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.scala new file mode 100644 index 0000000000..8f9fd63c34 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.scala @@ -0,0 +1,24 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.store.deviceinfo.DeviceInfo +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams +object PushToHomeUtil { + def getIbis2ModelValue( + deviceInfoOpt: Option[DeviceInfo], + target: Target, + stats: StatsReceiver + ): Option[Map[String, String]] = { + deviceInfoOpt.flatMap { deviceInfo => + val isAndroidEnabled = deviceInfo.isLandOnHomeAndroid && target.params( + PushFeatureSwitchParams.EnableTweetPushToHomeAndroid) + val isIOSEnabled = deviceInfo.isLandOnHomeiOS && target.params( + PushFeatureSwitchParams.EnableTweetPushToHomeiOS) + if (isAndroidEnabled || isIOSEnabled) { + stats.counter("enable_push_to_home").incr() + Some(Map("is_land_on_home" -> "true")) + } else None + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.scala new file mode 100644 index 0000000000..015e065ec0 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.scala @@ -0,0 +1,114 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.Counter +import com.twitter.finagle.stats.Stat +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.Invalid +import com.twitter.frigate.common.base.OK +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.refresh_handler.ResultWithDebugInfo +import com.twitter.frigate.pushservice.predicate.BigFilteringEpsilonGreedyExplorationPredicate +import com.twitter.frigate.pushservice.predicate.MlModelsHoldbackExperimentPredicate +import com.twitter.frigate.pushservice.take.candidate_validator.RFPHCandidateValidator +import com.twitter.frigate.pushservice.thriftscala.PushStatus +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.util.Future + +class RFPHTakeStepUtil()(globalStats: StatsReceiver) { + + implicit val statsReceiver: StatsReceiver = + globalStats.scope("RefreshForPushHandler") + private val takeStats: StatsReceiver = statsReceiver.scope("take") + private val notifierStats = takeStats.scope("notifier") + private val validatorStats = takeStats.scope("validator") + private val validatorLatency: Stat = validatorStats.stat("latency") + + private val executedPredicatesInTandem: Counter = + takeStats.counter("predicates_executed_in_tandem") + + private val bigFilteringEpsGreedyPredicate: NamedPredicate[PushCandidate] = + BigFilteringEpsilonGreedyExplorationPredicate()(takeStats) + private val bigFilteringEpsGreedyStats: StatsReceiver = + takeStats.scope("big_filtering_eps_greedy_predicate") + + private val modelPredicate: NamedPredicate[PushCandidate] = + MlModelsHoldbackExperimentPredicate()(takeStats) + private val mlPredicateStats: StatsReceiver = takeStats.scope("ml_predicate") + + private def updateFilteredStatusExptStats(candidate: PushCandidate, predName: String): Unit = { + + val recTypeStat = globalStats.scope( + candidate.commonRecType.toString + ) + + recTypeStat.counter(PushStatus.Filtered.toString).incr() + recTypeStat + .scope(PushStatus.Filtered.toString) + .counter(predName) + .incr() + } + + def isCandidateValid( + candidate: PushCandidate, + candidateValidator: RFPHCandidateValidator + ): Future[ResultWithDebugInfo] = { + val predResultFuture = Stat.timeFuture(validatorLatency) { + Future + .join( + bigFilteringEpsGreedyPredicate.apply(Seq(candidate)), + modelPredicate.apply(Seq(candidate)) + ).flatMap { + case (Seq(true), Seq(true)) => + executedPredicatesInTandem.incr() + + bigFilteringEpsGreedyStats + .scope(candidate.commonRecType.toString) + .counter("passed") + .incr() + + mlPredicateStats + .scope(candidate.commonRecType.toString) + .counter("passed") + .incr() + candidateValidator.validateCandidate(candidate).map((_, Nil)) + case (Seq(false), _) => + bigFilteringEpsGreedyStats + .scope(candidate.commonRecType.toString) + .counter("filtered") + .incr() + Future.value((Some(bigFilteringEpsGreedyPredicate), Nil)) + case (_, _) => + mlPredicateStats + .scope(candidate.commonRecType.toString) + .counter("filtered") + .incr() + Future.value((Some(modelPredicate), Nil)) + } + } + + predResultFuture.map { + case (Some(pred: NamedPredicate[_]), candPredicateResults) => + takeStats.counter("filtered_by_named_general_predicate").incr() + updateFilteredStatusExptStats(candidate, pred.name) + ResultWithDebugInfo( + Invalid(Some(pred.name)), + candPredicateResults + ) + + case (Some(_), candPredicateResults) => + takeStats.counter("filtered_by_unnamed_general_predicate").incr() + updateFilteredStatusExptStats(candidate, predName = "unk") + ResultWithDebugInfo( + Invalid(Some("unnamed_candidate_predicate")), + candPredicateResults + ) + + case (None, candPredicateResults) => + takeStats.counter("accepted_push_ok").incr() + ResultWithDebugInfo( + OK, + candPredicateResults + ) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.scala new file mode 100644 index 0000000000..8f24756ae7 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.scala @@ -0,0 +1,66 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.frigate.common.base.TweetAuthor +import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate +import com.twitter.hermit.predicate.socialgraph.Edge +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.socialgraph.thriftscala.RelationshipType + +/** + * This class provides utility functions for relationshipEdge for each Candidate type. + */ +object RelationshipUtil { + + /** + * Form relationEdges + * @param candidate PushCandidate + * @param relationship relationshipTypes for different candidate types + * @return relationEdges for different candidate types + */ + private def formRelationEdgeWithTargetIdAndAuthorId( + candidate: RawCandidate, + relationship: List[RelationshipType with Product] + ): List[RelationEdge] = { + candidate match { + case candidate: RawCandidate with TweetAuthor => + candidate.authorId match { + case Some(authorId) => + val edge = Edge(candidate.target.targetId, authorId) + for { + r <- relationship + } yield RelationEdge(edge, r) + case _ => List.empty[RelationEdge] + } + case _ => List.empty[RelationEdge] + } + } + + /** + * Form all relationshipEdges for basicTweetRelationShips + * @param candidate PushCandidate + * @return List of relationEdges for basicTweetRelationShips + */ + def getBasicTweetRelationships(candidate: RawCandidate): List[RelationEdge] = { + val relationship = List( + RelationshipType.DeviceFollowing, + RelationshipType.Blocking, + RelationshipType.BlockedBy, + RelationshipType.HideRecommendations, + RelationshipType.Muting) + formRelationEdgeWithTargetIdAndAuthorId(candidate, relationship) + } + + /** + * Form all relationshipEdges for F1tweetsRelationships + * @param candidate PushCandidate + * @return List of relationEdges for F1tweetsRelationships + */ + def getPreCandidateRelationshipsForInNetworkTweets( + candidate: RawCandidate + ): List[RelationEdge] = { + val relationship = List(RelationshipType.Following) + getBasicTweetRelationships(candidate) ++ formRelationEdgeWithTargetIdAndAuthorId( + candidate, + relationship) + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.scala new file mode 100644 index 0000000000..1b16ec8c05 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.scala @@ -0,0 +1,42 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.BroadcastStatsReceiver +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.thriftscala.PushResponse +import com.twitter.frigate.pushservice.thriftscala.PushStatus +import com.twitter.frigate.thriftscala.CommonRecommendationType + +object ResponseStatsTrackUtils { + def trackStatsForResponseToRequest( + crt: CommonRecommendationType, + target: Target, + response: PushResponse, + receivers: Seq[StatsReceiver] + )( + originalStats: StatsReceiver + ): Unit = { + val newReceivers = Seq( + originalStats + .scope("is_model_training_data") + .scope(target.isModelTrainingData.toString), + originalStats.scope("scribe_target").scope(IbisScribeTargets.crtToScribeTarget(crt)) + ) + + val broadcastStats = BroadcastStatsReceiver(receivers) + val broadcastStatsWithExpts = BroadcastStatsReceiver(newReceivers ++ receivers) + + if (response.status == PushStatus.Sent) { + if (target.isModelTrainingData) { + broadcastStats.counter("num_training_data_recs_sent").incr() + } + } + broadcastStatsWithExpts.counter(response.status.toString).incr() + if (response.status == PushStatus.Filtered) { + broadcastStats + .scope(response.status.toString) + .counter(response.filteredBy.getOrElse("None")) + .incr() + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.scala new file mode 100644 index 0000000000..4174fa21cb --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.scala @@ -0,0 +1,129 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.frigate.common.base.CandidateDetails +import com.twitter.frigate.common.base.CandidateResult +import com.twitter.frigate.common.base.Invalid +import com.twitter.frigate.common.base.OK +import com.twitter.frigate.common.base.Result +import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate +import com.twitter.frigate.pushservice.refresh_handler.ResultWithDebugInfo +import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPostCandidateValidator +import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPreCandidateValidator +import com.twitter.frigate.pushservice.thriftscala.PushStatus +import com.twitter.hermit.predicate.NamedPredicate +import com.twitter.util.Future + +class SendHandlerPredicateUtil()(globalStats: StatsReceiver) { + implicit val statsReceiver: StatsReceiver = + globalStats.scope("SendHandler") + private val validateStats: StatsReceiver = statsReceiver.scope("validate") + + private def updateFilteredStatusExptStats(candidate: PushCandidate, predName: String): Unit = { + + val recTypeStat = globalStats.scope( + candidate.commonRecType.toString + ) + + recTypeStat.counter(PushStatus.Filtered.toString).incr() + recTypeStat + .scope(PushStatus.Filtered.toString) + .counter(predName) + .incr() + } + + /** + * Parsing the candidateValidtor result into desired format for preValidation before ml filtering + * @param hydratedCandidates + * @param candidateValidator + * @return + */ + def preValidationForCandidate( + hydratedCandidates: Seq[CandidateDetails[PushCandidate]], + candidateValidator: SendHandlerPreCandidateValidator + ): Future[ + (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) + ] = { + val predResultFuture = + Future.collect( + hydratedCandidates.map(hydratedCandidate => + candidateValidator.validateCandidate(hydratedCandidate.candidate)) + ) + + predResultFuture.map { results => + results + .zip(hydratedCandidates) + .foldLeft( + ( + Seq.empty[CandidateDetails[PushCandidate]], + Seq.empty[CandidateResult[PushCandidate, Result]] + ) + ) { + case ((goodCandidates, filteredCandidates), (result, candidateDetails)) => + result match { + case None => + (goodCandidates :+ candidateDetails, filteredCandidates) + case Some(pred: NamedPredicate[_]) => + val r = Invalid(Some(pred.name)) + ( + goodCandidates, + filteredCandidates :+ CandidateResult[PushCandidate, Result]( + candidateDetails.candidate, + candidateDetails.source, + r + ) + ) + case Some(_) => + val r = Invalid(Some("Filtered by un-named predicate")) + ( + goodCandidates, + filteredCandidates :+ CandidateResult[PushCandidate, Result]( + candidateDetails.candidate, + candidateDetails.source, + r + ) + ) + } + } + } + } + + /** + * Parsing the candidateValidtor result into desired format for postValidation including and after ml filtering + * @param candidate + * @param candidateValidator + * @return + */ + def postValidationForCandidate( + candidate: PushCandidate, + candidateValidator: SendHandlerPostCandidateValidator + ): Future[ResultWithDebugInfo] = { + val predResultFuture = + candidateValidator.validateCandidate(candidate) + + predResultFuture.map { + case (Some(pred: NamedPredicate[_])) => + validateStats.counter("filtered_by_named_general_predicate").incr() + updateFilteredStatusExptStats(candidate, pred.name) + ResultWithDebugInfo( + Invalid(Some(pred.name)), + Nil + ) + + case Some(_) => + validateStats.counter("filtered_by_unnamed_general_predicate").incr() + updateFilteredStatusExptStats(candidate, predName = "unk") + ResultWithDebugInfo( + Invalid(Some("unnamed_candidate_predicate")), + Nil + ) + + case _ => + validateStats.counter("accepted_push_ok").incr() + ResultWithDebugInfo( + OK, + Nil + ) + } + } +} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.scala new file mode 100644 index 0000000000..a5b7cf2c52 --- /dev/null +++ b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.scala @@ -0,0 +1,340 @@ +package com.twitter.frigate.pushservice.util + +import com.twitter.contentrecommender.thriftscala.DisplayLocation +import com.twitter.finagle.stats.Stat +import com.twitter.frigate.common.base.TargetUser +import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.authorNotBeingFollowedPredicate +import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext +import com.twitter.frigate.pushservice.model.PushTypes.Target +import com.twitter.frigate.pushservice.model.PushTypes +import com.twitter.frigate.pushservice.store.UttEntityHydrationQuery +import com.twitter.frigate.pushservice.store.UttEntityHydrationStore +import com.twitter.hermit.predicate.Predicate +import com.twitter.hermit.predicate.socialgraph.RelationEdge +import com.twitter.interests.thriftscala.InterestRelationType +import com.twitter.interests.thriftscala.InterestRelationship +import com.twitter.interests.thriftscala.InterestedInInterestLookupContext +import com.twitter.interests.thriftscala.InterestedInInterestModel +import com.twitter.interests.thriftscala.ProductId +import com.twitter.interests.thriftscala.UserInterest +import com.twitter.interests.thriftscala.UserInterestData +import com.twitter.interests.thriftscala.UserInterests +import com.twitter.interests.thriftscala.{TopicListingViewerContext => TopicListingViewerContextCR} +import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult +import com.twitter.storehaus.ReadableStore +import com.twitter.timelines.configapi.Param +import com.twitter.topiclisting.TopicListingViewerContext +import com.twitter.topiclisting.utt.LocalizedEntity +import com.twitter.tsp.thriftscala.TopicListingSetting +import com.twitter.tsp.thriftscala.TopicSocialProofRequest +import com.twitter.tsp.thriftscala.TopicSocialProofResponse +import com.twitter.tsp.thriftscala.TopicWithScore +import com.twitter.util.Future +import scala.collection.Map + +case class TweetWithTopicProof( + tweetId: Long, + topicId: Long, + authorId: Option[Long], + score: Double, + tweetyPieResult: TweetyPieResult, + topicListingSetting: String, + algorithmCR: Option[String], + isOON: Boolean) + +object TopicsUtil { + + /** + * Obtains the Localized Entities for the provided SC Entity IDs + * @param target The target user for which we're obtaining candidates + * @param semanticCoreEntityIds The seq. of entity ids for which we would like to obtain the Localized Entities + * @param uttEntityHydrationStore Store to query the actual LocalizedEntities + * @return A Future Map consisting of the entity id as the key and LocalizedEntity as the value + */ + def getLocalizedEntityMap( + target: Target, + semanticCoreEntityIds: Set[Long], + uttEntityHydrationStore: UttEntityHydrationStore + ): Future[Map[Long, LocalizedEntity]] = { + buildTopicListingViewerContext(target) + .flatMap { topicListingViewerContext => + val query = UttEntityHydrationQuery(topicListingViewerContext, semanticCoreEntityIds.toSeq) + val localizedTopicEntitiesFut = + uttEntityHydrationStore.getLocalizedTopicEntities(query).map(_.flatten) + localizedTopicEntitiesFut.map { localizedTopicEntities => + localizedTopicEntities.map { localizedTopicEntity => + localizedTopicEntity.entityId -> localizedTopicEntity + }.toMap + } + } + } + + /** + * Fetch explict followed interests i.e Topics for targetUser + * + * @param targetUser: [[Target]] object representing a user eligible for MagicRecs notification + * @return: list of all Topics(Interests) Followed by targetUser + */ + def getTopicsFollowedByUser( + targetUser: Target, + interestsWithLookupContextStore: ReadableStore[ + InterestsLookupRequestWithContext, + UserInterests + ], + followedTopicsStats: Stat + ): Future[Option[Seq[UserInterest]]] = { + buildTopicListingViewerContext(targetUser).flatMap { topicListingViewerContext => + // explicit interests relation query + val explicitInterestsLookupRequest = InterestsLookupRequestWithContext( + targetUser.targetId, + Some( + InterestedInInterestLookupContext( + explicitContext = None, + inferredContext = None, + productId = Some(ProductId.Followable), + topicListingViewerContext = Some(topicListingViewerContext.toThrift), + disableExplicit = None, + disableImplicit = Some(true) + ) + ) + ) + + // filter explicit follow relationships from response + interestsWithLookupContextStore.get(explicitInterestsLookupRequest).map { + _.flatMap { userInterests => + val followedTopics = userInterests.interests.map { + _.filter { + case UserInterest(_, Some(interestData)) => + interestData match { + case UserInterestData.InterestedIn(interestedIn) => + interestedIn.exists { + case InterestedInInterestModel.ExplicitModel(explicitModel) => + explicitModel match { + case InterestRelationship.V1(v1) => + v1.relation == InterestRelationType.Followed + + case _ => false + } + + case _ => false + } + + case _ => false + } + + case _ => false // interestData unavailable + } + } + followedTopicsStats.add(followedTopics.getOrElse(Seq.empty[UserInterest]).size) + followedTopics + } + } + } + } + + /** + * + * @param target : [[Target]] object respresenting MagicRecs user + * + * @return: [[TopicListingViewerContext]] for querying topics + */ + def buildTopicListingViewerContext(target: Target): Future[TopicListingViewerContext] = { + Future.join(target.inferredUserDeviceLanguage, target.countryCode, target.targetUser).map { + case (inferredLanguage, countryCode, userInfo) => + TopicListingViewerContext( + userId = Some(target.targetId), + guestId = None, + deviceId = None, + clientApplicationId = None, + userAgent = None, + languageCode = inferredLanguage, + countryCode = countryCode, + userRoles = userInfo.flatMap(_.roles.map(_.roles.toSet)) + ) + } + } + + /** + * + * @param target : [[Target]] object respresenting MagicRecs user + * + * @return: [[TopicListingViewerContext]] for querying topics + */ + def buildTopicListingViewerContextForCR(target: Target): Future[TopicListingViewerContextCR] = { + TopicsUtil.buildTopicListingViewerContext(target).map(_.toThrift) + } + + /** + * + * @param target : [[Target]] object respresenting MagicRecs user + * @param tweets : [[Seq[TweetyPieResult]]] object representing Tweets to get TSP for + * @param topicSocialProofServiceStore: [[ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse]]] + * @param edgeStore: [[ReadableStore[RelationEdge, Boolean]]]] + * + * @return: [[Future[Seq[TweetWithTopicProof]]]] Tweets with topic proof + */ + def getTopicSocialProofs( + inputTarget: Target, + tweets: Seq[TweetyPieResult], + topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], + edgeStore: ReadableStore[RelationEdge, Boolean], + scoreThresholdParam: Param[Double] + ): Future[Seq[TweetWithTopicProof]] = { + buildTopicListingViewerContextForCR(inputTarget).flatMap { topicListingContext => + val tweetIds: Set[Long] = tweets.map(_.tweet.id).toSet + val tweetIdsToTweetyPie = tweets.map(tp => tp.tweet.id -> tp).toMap + val topicSocialProofRequest = + TopicSocialProofRequest( + inputTarget.targetId, + tweetIds, + DisplayLocation.MagicRecsRecommendTopicTweets, + TopicListingSetting.Followable, + topicListingContext) + + topicSocialProofServiceStore + .get(topicSocialProofRequest).flatMap { + case Some(topicSocialProofResponse) => + val topicProofCandidates = topicSocialProofResponse.socialProofs.collect { + case (tweetId, topicsWithScore) + if topicsWithScore.nonEmpty && topicsWithScore + .maxBy(_.score).score >= inputTarget + .params(scoreThresholdParam) => + // Get the topic with max score if there are any topics returned + val topicWithScore = topicsWithScore.maxBy(_.score) + TweetWithTopicProof( + tweetId, + topicWithScore.topicId, + tweetIdsToTweetyPie(tweetId).tweet.coreData.map(_.userId), + topicWithScore.score, + tweetIdsToTweetyPie(tweetId), + topicWithScore.topicFollowType.map(_.name).getOrElse(""), + topicWithScore.algorithmType.map(_.name), + isOON = true + ) + }.toSeq + + hydrateTopicProofCandidatesWithEdgeStore(inputTarget, topicProofCandidates, edgeStore) + case _ => Future.value(Seq.empty[TweetWithTopicProof]) + } + } + } + + /** + * Obtain TopicWithScores for provided tweet candidates and target + * @param target target user + * @param Tweets tweet candidates represented in a (tweetId, TweetyPieResult) map + * @param topicSocialProofServiceStore store to query topic social proof + * @param enableTopicAnnotation whether to enable topic annotation + * @param topicScoreThreshold threshold for topic score + * @return a (tweetId, TopicWithScore) map where the topic with highest topic score (if exists) is chosen + */ + def getTopicsWithScoreMap( + target: PushTypes.Target, + Tweets: Map[Long, Option[TweetyPieResult]], + topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], + enableTopicAnnotation: Boolean, + topicScoreThreshold: Double + ): Future[Option[Map[Long, TopicWithScore]]] = { + + if (enableTopicAnnotation) { + TopicsUtil + .buildTopicListingViewerContextForCR(target).flatMap { topicListingContext => + val tweetIds = Tweets.keySet + val topicSocialProofRequest = + TopicSocialProofRequest( + target.targetId, + tweetIds, + DisplayLocation.MagicRecsRecommendTopicTweets, + TopicListingSetting.Followable, + topicListingContext) + + topicSocialProofServiceStore + .get(topicSocialProofRequest).map { + _.map { topicSocialProofResponse => + topicSocialProofResponse.socialProofs + .collect { + case (tweetId, topicsWithScore) + if topicsWithScore.nonEmpty && Tweets(tweetId).nonEmpty + && topicsWithScore.maxBy(_.score).score >= topicScoreThreshold => + tweetId -> topicsWithScore.maxBy(_.score) + } + + } + } + } + } else { + Future.None + } + + } + + /** + * Obtain LocalizedEntities for provided tweet candidates and target + * @param target target user + * @param Tweets tweet candidates represented in a (tweetId, TweetyPieResult) map + * @param uttEntityHydrationStore store to query the actual LocalizedEntities + * @param topicSocialProofServiceStore store to query topic social proof + * @param enableTopicAnnotation whether to enable topic annotation + * @param topicScoreThreshold threshold for topic score + * @return a (tweetId, LocalizedEntity Option) Future map that stores Localized Entity (can be empty) for given tweetId + */ + def getTweetIdLocalizedEntityMap( + target: PushTypes.Target, + Tweets: Map[Long, Option[TweetyPieResult]], + uttEntityHydrationStore: UttEntityHydrationStore, + topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], + enableTopicAnnotation: Boolean, + topicScoreThreshold: Double + ): Future[Map[Long, Option[LocalizedEntity]]] = { + + val topicWithScoreMap = getTopicsWithScoreMap( + target, + Tweets, + topicSocialProofServiceStore, + enableTopicAnnotation, + topicScoreThreshold) + + topicWithScoreMap.flatMap { topicWithScores => + topicWithScores match { + case Some(topics) => + val topicIds = topics.collect { case (_, topic) => topic.topicId }.toSet + val LocalizedEntityMapFut = + getLocalizedEntityMap(target, topicIds, uttEntityHydrationStore) + + LocalizedEntityMapFut.map { LocalizedEntityMap => + topics.map { + case (tweetId, topic) => + tweetId -> LocalizedEntityMap.get(topic.topicId) + } + } + case _ => Future.value(Map[Long, Option[LocalizedEntity]]()) + } + } + + } + + /** + * Hydrate TweetWithTopicProof candidates with isOON field info, + * based on the following relationship between target user and candidate author in edgeStore + * @return TweetWithTopicProof candidates with isOON field populated + */ + def hydrateTopicProofCandidatesWithEdgeStore( + inputTarget: TargetUser, + topicProofCandidates: Seq[TweetWithTopicProof], + edgeStore: ReadableStore[RelationEdge, Boolean], + ): Future[Seq[TweetWithTopicProof]] = { + // IDs of all authors of TopicProof candidates that are OON with respect to inputTarget + val validOONAuthorIdsFut = + Predicate.filter( + topicProofCandidates.flatMap(_.authorId).distinct, + authorNotBeingFollowedPredicate(inputTarget, edgeStore)) + + validOONAuthorIdsFut.map { validOONAuthorIds => + topicProofCandidates.map(candidate => { + candidate.copy(isOON = + candidate.authorId.isDefined && validOONAuthorIds.contains(candidate.authorId.get)) + }) + } + } + +}