From 85041c84b9fd0a5c3b392f97d577a35f3bdf0d5b Mon Sep 17 00:00:00 2001 From: Petru-Mugurel Lingurar Date: Fri, 8 May 2020 15:48:01 +0000 Subject: [PATCH] Bug 1633568 - Introduce a installation ping This mobile-specific ping is intended to keep track of installs and Adjust attribution. The app should send two installation pings, based on different reasons: One sent immediately after the app starts. One sent immediately after the Adjust attribution data becomes available. If the app is later deleted and installed again, the installation pings must be sent again. This will will be a modern ping, sent to hostname/submit/mobile/docType/appVersion/docId. Differential Revision: https://phabricator.services.mozilla.com/D74073 --- .../java/org/mozilla/gecko/BrowserApp.java | 5 +- .../TelemetryInstallationPingDelegate.java | 80 ++++++++ .../TelemetryInstallationPingBuilder.java | 181 ++++++++++++++++++ .../TelemetryInstallationPingStore.java | 117 +++++++++++ .../stores/TelemetryJSONFilePingStore.java | 16 +- 5 files changed, 392 insertions(+), 7 deletions(-) create mode 100644 mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryInstallationPingDelegate.java create mode 100644 mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetryInstallationPingBuilder.java create mode 100644 mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryInstallationPingStore.java diff --git a/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java b/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java index 9fda26104c239..04bde0da3c5ba 100644 --- a/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java +++ b/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java @@ -141,6 +141,7 @@ import org.mozilla.gecko.tabs.TabHistoryFragment; import org.mozilla.gecko.tabs.TabHistoryPage; import org.mozilla.gecko.tabs.TabsPanel; import org.mozilla.gecko.telemetry.TelemetryCorePingDelegate; +import org.mozilla.gecko.telemetry.TelemetryInstallationPingDelegate; import org.mozilla.gecko.telemetry.TelemetryUploadService; import org.mozilla.gecko.telemetry.measurements.SearchCountMeasurements; import org.mozilla.gecko.telemetry.TelemetryActivationPingDelegate; @@ -324,6 +325,7 @@ public class BrowserApp extends GeckoApp private final TelemetryCorePingDelegate mTelemetryCorePingDelegate = new TelemetryCorePingDelegate(); private final TelemetryActivationPingDelegate mTelemetryActivationPingDelegate = new TelemetryActivationPingDelegate(); + private final TelemetryInstallationPingDelegate mTelemetryInstallationPingDelegate = new TelemetryInstallationPingDelegate(); private final List delegates = Collections.unmodifiableList(Arrays.asList( new ScreenshotDelegate(), @@ -331,8 +333,9 @@ public class BrowserApp extends GeckoApp new ReaderViewBookmarkPromotion(), mTelemetryCorePingDelegate, mTelemetryActivationPingDelegate, + mTelemetryInstallationPingDelegate, new OfflineTabStatusDelegate(), - new AdjustBrowserAppDelegate(mTelemetryCorePingDelegate) + new AdjustBrowserAppDelegate(mTelemetryCorePingDelegate, mTelemetryInstallationPingDelegate) )); @NonNull diff --git a/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryInstallationPingDelegate.java b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryInstallationPingDelegate.java new file mode 100644 index 0000000000000..50b8c068ca333 --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryInstallationPingDelegate.java @@ -0,0 +1,80 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package org.mozilla.gecko.telemetry; + +import android.support.annotation.NonNull; +import android.util.Log; + +import com.adjust.sdk.AdjustAttribution; + +import org.mozilla.gecko.BrowserApp; +import org.mozilla.gecko.GeckoAppShell; +import org.mozilla.gecko.adjust.AttributionHelperListener; +import org.mozilla.gecko.delegates.BrowserAppDelegate; +import org.mozilla.gecko.telemetry.pingbuilders.TelemetryInstallationPingBuilder; +import org.mozilla.gecko.telemetry.pingbuilders.TelemetryInstallationPingBuilder.PingReason; +import org.mozilla.gecko.telemetry.schedulers.TelemetryUploadAllPingsImmediatelyScheduler; +import org.mozilla.gecko.telemetry.stores.TelemetryInstallationPingStore; +import org.mozilla.gecko.util.ThreadUtils; + +import java.io.IOException; + +public class TelemetryInstallationPingDelegate + extends BrowserAppDelegate + implements AttributionHelperListener { + + private static final String LOGTAG = "InstallPingDelegate"; + + @Override + public void onStart(BrowserApp browserApp) { + if (!TelemetryUploadService.isUploadEnabledByAppConfig(browserApp)) { + return; + } + + if (!TelemetryInstallationPingStore.hasLightPingBeenUploaded()) { + ThreadUtils.postToBackgroundThread(() -> { + TelemetryInstallationPingStore store = new TelemetryInstallationPingStore(); + TelemetryOutgoingPing ping = new TelemetryInstallationPingBuilder() + .setReason(PingReason.APP_STARTED) + .build(); + + try { + store.storePing(ping.getPayload(), PingReason.APP_STARTED.value, ping.getURLPath()); + store.queuePingsForUpload(new TelemetryUploadAllPingsImmediatelyScheduler()); + } catch (IOException e) { + // #storePing() might throw. Nothing to do. Will try again later. + Log.w(LOGTAG, "Could not store ping. Will try again later"); + } + }); + } + } + + @Override + public void onAttributionChanged(@NonNull final AdjustAttribution attribution) { + if (!TelemetryUploadService.isUploadEnabledByAppConfig(GeckoAppShell.getApplicationContext())) { + return; + } + + if (!TelemetryInstallationPingStore.hasFullPingBeenUploaded()) { + ThreadUtils.postToBackgroundThread(() -> { + TelemetryInstallationPingStore store = new TelemetryInstallationPingStore(); + TelemetryOutgoingPing ping = new TelemetryInstallationPingBuilder() + .setReason(PingReason.ADJUST_AVAILABLE) + .setAdjustProperties(attribution) + .build(); + + try { + store.storePing(ping.getPayload(), PingReason.ADJUST_AVAILABLE.value, ping.getURLPath()); + store.queuePingsForUpload(new TelemetryUploadAllPingsImmediatelyScheduler()); + } catch (IOException e) { + // #storePing() might throw. Nothing to do. Will try again later. + Log.w(LOGTAG, "Could not store ping. Will try again later"); + } + }); + } + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetryInstallationPingBuilder.java b/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetryInstallationPingBuilder.java new file mode 100644 index 0000000000000..8638b23858497 --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetryInstallationPingBuilder.java @@ -0,0 +1,181 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package org.mozilla.gecko.telemetry.pingbuilders; + +import android.content.Context; +import android.content.SharedPreferences; +import android.os.Build; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; +import android.util.Log; + +import com.adjust.sdk.AdjustAttribution; + +import org.mozilla.gecko.AppConstants; +import org.mozilla.gecko.GeckoAppShell; +import org.mozilla.gecko.GeckoSharedPrefs; +import org.mozilla.gecko.GeckoThread; +import org.mozilla.gecko.Locales; +import org.mozilla.gecko.util.DateUtil; +import org.mozilla.gecko.util.HardwareUtils; + +import java.lang.reflect.Method; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Locale; + +public class TelemetryInstallationPingBuilder extends TelemetryPingBuilder { + private static final String LOGTAG = "InstallPingBuilder"; + + public enum PingReason { + APP_STARTED("app-started"), + ADJUST_AVAILABLE("adjust-available"); + + PingReason(String reason) { + this.value = reason; + } + + public final String value; + } + + private static final String PING_TYPE = "installation"; + + private static final String PREF_KEY_SEQ_NUMBER = "installationPingSeqNumber"; + + private static final String REASON = "reason"; + private static final String PING_QUEUED_TIMES = "seq"; + private static final String CLIENT_ID = "client_id"; + private static final String DEVICE_ID = "device_id"; + private static final String LOCALE = "locale"; + private static final String OS_NAME = "os"; + private static final String OS_VERSION = "osversion"; + private static final String DEVICE_MANUFACTURER = "manufacturer"; + private static final String DEVICE_MODEL = "model"; + private static final String DEVICE_ABI = "arch"; + private static final String PROFILE_DATE = "profile_date"; + private static final String PING_CREATION_TIME = "created"; + private static final String TIMEZONE_OFFSET = "tz"; + private static final String APP_NAME = "app_name"; + private static final String RELEASE_CHANNEL = "channel"; + private static final String ADJUST_CAMPAIGN = "campaign"; + private static final String ADJUST_ADGROUP = "adgroup"; + private static final String ADJUST_CREATIVE = "creative"; + private static final String ADJUST_NETWORK = "network"; + + public TelemetryInstallationPingBuilder() { + super(UNIFIED_TELEMETRY_VERSION, false); + setPayloadConstants(); + } + + @Override + public String getDocType() { + return PING_TYPE; + } + + @Override + public String[] getMandatoryFields() { + return new String[]{ + REASON, + PING_QUEUED_TIMES, + CLIENT_ID, + DEVICE_ID, + LOCALE, + OS_NAME, + OS_VERSION, + DEVICE_MANUFACTURER, + DEVICE_MODEL, + DEVICE_ABI, + PROFILE_DATE, + PING_CREATION_TIME, + TIMEZONE_OFFSET, + APP_NAME, + RELEASE_CHANNEL, + }; + } + + public @NonNull TelemetryInstallationPingBuilder setReason(@NonNull PingReason reason) { + payload.put(REASON, reason.value); + + return this; + } + + public @NonNull TelemetryInstallationPingBuilder setAdjustProperties(@NonNull final AdjustAttribution attribution) { + payload.put(ADJUST_CAMPAIGN, attribution.campaign); + payload.put(ADJUST_ADGROUP, attribution.adgroup); + payload.put(ADJUST_CREATIVE, attribution.creative); + payload.put(ADJUST_NETWORK, attribution.network); + + return this; + } + + private void setPayloadConstants() { + payload.put(PING_QUEUED_TIMES, incrementAndGetQueueTimes()); + payload.put(CLIENT_ID, getGeckoClientID()); + payload.put(DEVICE_ID, getAdvertisingId()); + payload.put(LOCALE, Locales.getLanguageTag(Locale.getDefault())); + payload.put(OS_NAME, TelemetryPingBuilder.OS_NAME); + payload.put(OS_VERSION, Integer.toString(Build.VERSION.SDK_INT)); + payload.put(DEVICE_MANUFACTURER, Build.MANUFACTURER); + payload.put(DEVICE_MODEL, Build.MODEL); + payload.put(DEVICE_ABI, HardwareUtils.getRealAbi()); + payload.put(PROFILE_DATE, getGeckoProfileCreationDate()); + payload.put(PING_CREATION_TIME, new SimpleDateFormat("yyyy-MM-dd", Locale.US).format(new Date())); + payload.put(TIMEZONE_OFFSET, DateUtil.getTimezoneOffsetInMinutesForGivenDate(Calendar.getInstance())); + payload.put(APP_NAME, AppConstants.MOZ_APP_BASENAME); + payload.put(RELEASE_CHANNEL, AppConstants.MOZ_UPDATE_CHANNEL); + } + + private @Nullable String getGeckoClientID() { + // zero-ed Gecko profile that respects the expected format "8-4-4-4-12" chars + String clientID = "00000000-0000-0000-0000-000000000000"; + try { + clientID = GeckoThread.getActiveProfile().getClientId(); + } catch (Exception e) { + Log.w(LOGTAG, "Could not get Gecko Client ID", e); + } + + return clientID; + } + + private @Nullable String getAdvertisingId() { + String advertisingId = null; + try { + final Class clazz = Class.forName("org.mozilla.gecko.advertising.AdvertisingUtil"); + final Method getAdvertisingId = clazz.getMethod("getAdvertisingId", Context.class); + advertisingId = (String) getAdvertisingId.invoke(clazz, GeckoAppShell.getApplicationContext()); + } catch (Exception e) { + Log.w(LOGTAG, "Could not get advertising ID", e); + } + + return advertisingId; + } + + private int incrementAndGetQueueTimes() { + final SharedPreferences sharedPrefs = GeckoSharedPrefs.forProfile(GeckoAppShell.getApplicationContext()); + + // 1-based, always incremented + final int incrementedSeqNumber = sharedPrefs.getInt(PREF_KEY_SEQ_NUMBER, 0) + 1; + sharedPrefs.edit().putInt(PREF_KEY_SEQ_NUMBER, incrementedSeqNumber).apply(); + + return incrementedSeqNumber; + } + + private int getGeckoProfileCreationDate() { + // The method returns days since epoch. An int is enough. + int date = 0; + try { + date = TelemetryActivationPingBuilder.getProfileCreationDate( + GeckoAppShell.getApplicationContext(), + GeckoThread.getActiveProfile()).intValue(); + } catch (NullPointerException e) { + Log.w(LOGTAG, "Could not get Gecko profile creation date", e); + } + + return date; + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryInstallationPingStore.java b/mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryInstallationPingStore.java new file mode 100644 index 0000000000000..9a2a2f1962208 --- /dev/null +++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryInstallationPingStore.java @@ -0,0 +1,117 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package org.mozilla.gecko.telemetry.stores; + +import android.content.SharedPreferences; +import android.os.Parcel; +import android.os.Parcelable; +import android.support.annotation.NonNull; + +import org.mozilla.gecko.GeckoAppShell; +import org.mozilla.gecko.GeckoSharedPrefs; +import org.mozilla.gecko.GeckoThread; +import org.mozilla.gecko.telemetry.schedulers.TelemetryUploadAllPingsImmediatelyScheduler; +import org.mozilla.gecko.telemetry.pingbuilders.TelemetryInstallationPingBuilder.PingReason; +import org.mozilla.gecko.util.FileUtils; + +import java.io.File; +import java.io.FilenameFilter; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class TelemetryInstallationPingStore extends TelemetryJSONFilePingStore { + private static final String PREF_KEY_WAS_LIGHT_PING_SENT = "wasLightInstallationPingSent"; + private static final String PREF_KEY_WAS_FULL_PING_SENT = "wasFullInstallationPingSent"; + private static final String INSTALLATION_PING_STORE_DIR = "installation_ping"; + private static final String DEFAULT_PROFILE = "default"; + + public TelemetryInstallationPingStore() { + super(getInstallationPingStoreDir(), getCurrentProfileName()); + } + + @Override + public void onUploadAttemptComplete(@NonNull final Set successfulRemoveIDs) { + // Delete the just uploaded files + super.onUploadAttemptComplete(successfulRemoveIDs); + + // Remember the uploads. We only wanted one of each. + if (successfulRemoveIDs.contains(PingReason.APP_STARTED.value)) { + setLightPingUploaded(); + } + if (successfulRemoveIDs.contains(PingReason.ADJUST_AVAILABLE.value)) { + setFullPingUploaded(); + } + } + + @Override + protected FilenameFilter getFilenameFilter() { + return new FileUtils.FilenameWhitelistFilter( + new HashSet<>(Arrays.asList(PingReason.APP_STARTED.value, PingReason.ADJUST_AVAILABLE.value)) + ); + } + + public void queuePingsForUpload(@NonNull final TelemetryUploadAllPingsImmediatelyScheduler scheduler) { + scheduler.scheduleUpload(GeckoAppShell.getApplicationContext(), this); + } + + public static boolean hasLightPingBeenUploaded() { + return getSharedPrefs().getBoolean(PREF_KEY_WAS_LIGHT_PING_SENT, false); + } + + public static boolean hasFullPingBeenUploaded() { + return getSharedPrefs().getBoolean(PREF_KEY_WAS_FULL_PING_SENT, false); + } + + private static void setLightPingUploaded() { + getSharedPrefs().edit().putBoolean(PREF_KEY_WAS_LIGHT_PING_SENT, true).apply(); + } + + private static void setFullPingUploaded() { + getSharedPrefs().edit().putBoolean(PREF_KEY_WAS_FULL_PING_SENT, true).apply(); + } + + private static @NonNull SharedPreferences getSharedPrefs() { + return GeckoSharedPrefs.forProfile(GeckoAppShell.getApplicationContext()); + } + + private static @NonNull File getInstallationPingStoreDir() { + return GeckoAppShell.getApplicationContext().getFileStreamPath(INSTALLATION_PING_STORE_DIR); + } + + private static @NonNull String getCurrentProfileName() { + return GeckoThread.getActiveProfile() != null ? + GeckoThread.getActiveProfile().getName() : + DEFAULT_PROFILE; + } + + + // Class needs to be Parcelable as it will be passed through Intents + public static final Parcelable.Creator CREATOR = + new Parcelable.Creator() { + + @Override + public TelemetryInstallationPingStore createFromParcel(final Parcel source) { + return new TelemetryInstallationPingStore(); + } + + @Override + public TelemetryInstallationPingStore[] newArray(final int size) { + return new TelemetryInstallationPingStore[size]; + } + }; + + @Override + public int describeContents() { + return 0; + } + + @Override + public void writeToParcel(final Parcel dest, final int flags) { + super.writeToParcel(dest, flags); + } +} diff --git a/mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryJSONFilePingStore.java b/mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryJSONFilePingStore.java index 65e1b1bdb4dd2..6efacc79fcd33 100644 --- a/mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryJSONFilePingStore.java +++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryJSONFilePingStore.java @@ -75,7 +75,7 @@ public class TelemetryJSONFilePingStore extends TelemetryPingStore { @VisibleForTesting static final String KEY_URL_PATH = "u"; private final File storeDir; - private final FilenameFilter uuidFilenameFilter; + private final FilenameFilter filenameFilter; private final FileLastModifiedComparator fileLastModifiedComparator = new FileLastModifiedComparator(); @WorkerThread // Writes to disk @@ -89,7 +89,7 @@ public class TelemetryJSONFilePingStore extends TelemetryPingStore { this.storeDir = storeDir; this.storeDir.mkdirs(); - uuidFilenameFilter = new FilenameRegexFilter(UUIDUtil.UUID_PATTERN); + filenameFilter = getFilenameFilter(); if (!this.storeDir.canRead() || !this.storeDir.canWrite() || !this.storeDir.canExecute()) { throw new IllegalStateException("Cannot read, write, or execute store dir: " + @@ -125,7 +125,7 @@ public class TelemetryJSONFilePingStore extends TelemetryPingStore { @Override public void maybePrunePings() { - final File[] files = storeDir.listFiles(uuidFilenameFilter); + final File[] files = storeDir.listFiles(filenameFilter); if (files == null) { return; } @@ -157,7 +157,7 @@ public class TelemetryJSONFilePingStore extends TelemetryPingStore { @Override public ArrayList getAllPings() { - final File[] fileArray = storeDir.listFiles(uuidFilenameFilter); + final File[] fileArray = storeDir.listFiles(filenameFilter); if (fileArray == null) { // Intentionally don't log all info for the store directory to prevent leaking the path. Log.w(LOGTAG, "listFiles unexpectedly returned null - unable to retrieve pings. Debug: exists? " + @@ -195,7 +195,7 @@ public class TelemetryJSONFilePingStore extends TelemetryPingStore { @Override public int getCount() { - final File[] fileArray = storeDir.listFiles(uuidFilenameFilter); + final File[] fileArray = storeDir.listFiles(filenameFilter); if (fileArray == null) { Log.w(LOGTAG, "listFiles unexpectedly returned null - unable to retrieve pings. Assuming 0. " + "Debug: exists? " + storeDir.exists() + "; directory? " + storeDir.isDirectory()); @@ -207,7 +207,7 @@ public class TelemetryJSONFilePingStore extends TelemetryPingStore { @Override public Set getStoredIDs() { final Set ids = new HashSet<>(); - final File[] fileArray = storeDir.listFiles(uuidFilenameFilter); + final File[] fileArray = storeDir.listFiles(filenameFilter); if (fileArray == null) { return ids; } @@ -315,6 +315,10 @@ public class TelemetryJSONFilePingStore extends TelemetryPingStore { } } + protected FilenameFilter getFilenameFilter() { + return new FilenameRegexFilter(UUIDUtil.UUID_PATTERN); + } + public static final Parcelable.Creator CREATOR = new Parcelable.Creator() { @Override public TelemetryJSONFilePingStore createFromParcel(final Parcel source) { -- GitLab