feat(grfn/bbbg): Allow importing event attendees

Add support for importing the tsv that meetup exports into a list of
upserted attendees, and event-attendee joins.

Change-Id: I5f4ddc9fc63bcc6b0334bc3e1d3cbc4d5b99c21b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4570
Tested-by: BuildkiteCI
Reviewed-by: grfn <grfn@gws.fyi>
Autosubmit: grfn <grfn@gws.fyi>

users/grfn/bbbg/deps.edn

@@ -21,8 +21,6 @@
   hiccup/hiccup {:mvn/version "1.0.5"}
   garden/garden {:mvn/version "1.3.10"}
 
-  ;; Utils
-  com.stuartsierra/component {:mvn/version "1.0.0"}
 
   ;; Logging + Observability
   ch.qos.logback/logback-classic {:mvn/version "1.2.3"
@@ -37,10 +35,12 @@
   clj-commons/iapetos {:mvn/version "0.1.12"}
 
   ;; Utilities
+  com.stuartsierra/component {:mvn/version "1.0.0"}
   yogthos/config {:mvn/version "1.1.8"}
   clojure.java-time/clojure.java-time {:mvn/version "0.3.3"}
   cheshire/cheshire {:mvn/version "5.10.1"}
   org.apache.commons/commons-lang3 {:mvn/version "3.11"}
+  org.clojure/data.csv {:mvn/version "1.0.0"}
 
   ;; Spec
   org.clojure/spec.alpha {:mvn/version "0.3.214"}

users/grfn/bbbg/deps.nix

@@ -847,6 +847,19 @@ let repos = [
     paths = [ src ];
   }
 
+  rec {
+    name = "data.csv/org.clojure";
+    src = fetchMavenArtifact {
+      inherit repos;
+      artifactId = "data.csv";
+      groupId = "org.clojure";
+      sha512 = "b039775a859ed27eca8f8ae74ccb6afde3ad1fe2b3cbe542240c324d60fe1237e495eb1300ee9eb4ff4ef59f01faf7aec6ef1dd6a025ee4fe556c1d91acfcf1b";
+      version = "1.0.0";
+      
+    };
+    paths = [ src ];
+  }
+
   rec {
     name = "simpleclient_tracer_otel_agent/io.prometheus";
     src = fetchMavenArtifact {

users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.down.sql

@@ -0,0 +1 @@
+drop index attendee_uniq_meetup_user_id;

users/grfn/bbbg/resources/migrations/20211224161028-add-attendee-unique-meetup-id.up.sql

@@ -0,0 +1,2 @@
+create unique index "attendee_uniq_meetup_user_id" on attendee (meetup_user_id);
+-- ;;

users/grfn/bbbg/src/bbbg/attendee.clj

@@ -3,10 +3,8 @@
 
 (s/def ::id uuid?)
 
-(s/def ::meetup-name string?)
+(s/def ::meetup-name (s/and string? seq))
 
 (s/def ::discord-name (s/nilable string?))
 
-(s/def ::meetup-user-id (s/nilable string?))
-
 (s/def ::organizer-notes string?)

users/grfn/bbbg/src/bbbg/db.clj

@@ -355,6 +355,7 @@
 
 (comment
   (def db (:db bbbg.core/system))
-  (generate-migration db "add-attendee-checks")
+  (generate-migration db "add-attendee-unique-meetup-id")
   (migrate! db)
+
   )

users/grfn/bbbg/src/bbbg/db/attendee.clj

@@ -6,7 +6,8 @@
    honeysql-postgres.helpers
    [honeysql.helpers
     :refer
-    [merge-group-by merge-join merge-left-join merge-select merge-where]]))
+    [merge-group-by merge-join merge-left-join merge-select merge-where]]
+   [bbbg.util.core :as u]))
 
 (defn search
   ([q] (search {:select [:attendee.*] :from [:attendee]} q))
@@ -41,6 +42,19 @@
                        [:not :event_attendee.attended]])
          :no-shows]))))
 
+(defn upsert-all!
+  [db attendees]
+  (db/list
+   db
+   {:insert-into :attendee
+    :values (map #(->> %
+                       (db/process-key-map :attendee)
+                       (u/map-keys keyword))
+                 attendees)
+    :upsert {:on-conflict [:meetup-user-id]
+             :do-update-set [:meetup-name]}
+    :returning [:id :meetup-user-id]}))
+
 (comment
   (def db (:db bbbg.core/system))
   (db/database? db)

users/grfn/bbbg/src/bbbg/db/attendee_check.clj

@@ -8,28 +8,31 @@
 
 (defn attendees-with-last-checks
   [db attendees]
-  (let [ids (map ::attendee/id attendees)
+  (when (seq attendees)
-        checks
+    (let [ids (map ::attendee/id attendees)
-        (db/list db {:select [:attendee-check.*]
+          checks
-                     :from [:attendee-check]
+          (db/list db {:select [:attendee-check.*]
-                     :join [[{:select [:%max.attendee-check.checked-at
+                       :from [:attendee-check]
-                                       :attendee-check.attendee-id]
+                       :join [[{:select [:%max.attendee-check.checked-at
-                              :from [:attendee-check]
+                                         :attendee-check.attendee-id]
-                              :group-by [:attendee-check.attendee-id]
+                                :from [:attendee-check]
-                              :where [:in :attendee-check.attendee-id ids]}
+                                :group-by [:attendee-check.attendee-id]
-                             :last-check]
+                                :where [:in :attendee-check.attendee-id ids]}
-                            [:=
+                               :last-check]
-                             :attendee-check.attendee-id
+                              [:=
-                             :last-check.attendee-id]]})
+                               :attendee-check.attendee-id
-        users (u/key-by
+                               :last-check.attendee-id]]})
-               ::user/id
+          users (if (seq checks)
-               (db/list db {:select [:public.user.*]
+                  (u/key-by
-                            :from [:public.user]
+                   ::user/id
-                            :where [:in :id (map ::user/id checks)]}))
+                   (db/list db {:select [:public.user.*]
-        checks (map #(assoc % :user (users (::user/id %))) checks)
+                                :from [:public.user]
-        attendee-id->check (u/key-by ::attendee/id checks)]
+                                :where [:in :id (map ::user/id checks)]}))
-    (map #(assoc % :last-check (attendee-id->check (::attendee/id %)))
+                  {})
-         attendees)))
+          checks (map #(assoc % :user (users (::user/id %))) checks)
+          attendee-id->check (u/key-by ::attendee/id checks)]
+      (map #(assoc % :last-check (attendee-id->check (::attendee/id %)))
+           attendees))))
 
 (comment
   (def db (:db bbbg.core/system))

users/grfn/bbbg/src/bbbg/db/event_attendee.clj

@@ -0,0 +1,16 @@
+(ns bbbg.db.event-attendee
+  (:require honeysql-postgres.format
+            [bbbg.db :as db]
+            [bbbg.util.core :as u]))
+
+(defn upsert-all!
+  [db attendees]
+  (db/execute!
+   db
+   {:insert-into :event-attendee
+    :values (map #(->> %
+                       (db/process-key-map :event-attendee)
+                       (u/map-keys keyword))
+                 attendees)
+    :upsert {:on-conflict [:event-id :attendee-id]
+             :do-update-set [:rsvpd-attending]}}))

users/grfn/bbbg/src/bbbg/event_attendee.clj

@@ -2,3 +2,5 @@
   (:require [clojure.spec.alpha :as s]))
 
 (s/def ::attended? boolean?)
+
+(s/def ::rsvpd-attending? boolean?)

users/grfn/bbbg/src/bbbg/meetup/import.clj

@@ -0,0 +1,124 @@
+(ns bbbg.meetup.import
+  (:require
+   [bbbg.attendee :as attendee]
+   [bbbg.db.attendee :as db.attendee]
+   [bbbg.db.event-attendee :as db.event-attendee]
+   [bbbg.event :as event]
+   [bbbg.event-attendee :as event-attendee]
+   [bbbg.meetup-user :as meetup-user]
+   [bbbg.util.core :as u]
+   [bbbg.util.spec :as u.s]
+   [clojure.data.csv :as csv]
+   [clojure.java.io :as io]
+   [clojure.spec.alpha :as s]
+   [clojure.string :as str]
+   [expound.alpha :as exp]))
+
+(def spreadsheet-column->key
+  {"Name" :name
+   "User ID" :user-id
+   "Title" :title
+   "Event Host" :event-host
+   "RSVP" :rsvp
+   "Guests" :guests
+   "RSVPed on" :rsvped-on
+   "Joined Group on" :joined-group-on
+   "URL of Member Profile" :member-profile-url})
+
+(defn read-attendees [f]
+  (with-open [reader (io/reader f)]
+    (let [[headers & rows] (-> reader (csv/read-csv :separator \tab))
+          keys (map spreadsheet-column->key headers)]
+      (doall
+       (->> rows
+            (map (partial zipmap keys))
+            (map (partial u/filter-kv (fn [k _] (some? k))))
+            (filter (partial some (comp seq val))))))))
+
+;;;
+
+(s/def ::imported-attendee
+  (s/keys :req [::attendee/meetup-name
+                ::meetup-user/id]))
+
+(def key->attendee-col
+  {:name ::attendee/meetup-name
+   :user-id ::meetup-user/id})
+
+(defn row-user-id->user-id [row-id]
+  (str/replace-first row-id "user " ""))
+
+(defn check-attendee [attendee]
+  ()
+  (if (s/valid? ::imported-attendee attendee)
+    attendee
+    (throw (ex-info
+            (str "Invalid imported attendee\n"
+                 (exp/expound-str ::imported-attendee attendee))
+            (assoc (s/explain-data ::imported-attendee attendee)
+                   ::s/failure
+                   ::s/assertion-failed)))))
+
+(defn row->attendee [r]
+  (u.s/assert!
+   ::imported-attendee
+   (update (u/keep-keys key->attendee-col r)
+           ::meetup-user/id row-user-id->user-id)))
+
+;;;
+
+(s/def ::imported-event-attendee
+  (s/keys :req [::event-attendee/rsvpd-attending?
+                ::attendee/id
+                ::event/id]))
+
+(def key->event-attendee-col
+  {:rsvp ::event-attendee/rsvpd-attending?})
+
+(defn row->event-attendee
+  [{event-id ::event/id :keys [meetup-id->attendee-id]} r]
+  (let [attendee-id (-> r :user-id row-user-id->user-id meetup-id->attendee-id)]
+    (u.s/assert!
+     ::imported-event-attendee
+     (-> (u/keep-keys key->event-attendee-col r)
+         (update ::event-attendee/rsvpd-attending?
+                 (partial = "Yes"))
+         (assoc ::event/id event-id
+                ::attendee/id attendee-id)))))
+
+;;;
+
+(defn import-data! [db event-id f]
+  (let [rows (read-attendees f)
+        attendees (db.attendee/upsert-all! db (map row->attendee rows))
+        meetup-id->attendee-id (into {}
+                                     (map (juxt ::meetup-user/id ::attendee/id))
+                                     attendees)]
+    (db.event-attendee/upsert-all!
+     db
+     (map (partial row->event-attendee
+                   {::event/id event-id
+                    :meetup-id->attendee-id meetup-id->attendee-id})
+          rows))))
+
+;;; Spreadsheet columns:
+;;;
+;;; Name
+;;; User ID
+;;; Title
+;;; Event Host
+;;; RSVP
+;;; Guests
+;;; RSVPed on
+;;; Joined Group on
+;;; URL of Member Profile
+;;; Have you been to one of our events before? Note, attendance at all events will require proof of vaccination until further notice.
+
+(comment
+  (def -filename- "/home/grfn/code/depot/users/grfn/bbbg/sample-data.tsv")
+  (def event-id #uuid "09f8fed6-7480-451b-89a2-bb4edaeae657")
+
+  (read-attendees -filename-)
+  (import-data! (:db bbbg.core/system) event-id -filename-)
+
+  )

users/grfn/bbbg/src/bbbg/meetup_user.clj

@@ -0,0 +1,8 @@
+(ns bbbg.meetup-user
+  (:require [clojure.spec.alpha :as s]))
+
+(s/def ::id
+  (s/nilable
+   (s/and string?
+          seq
+          #(re-matches #"\d+" %))))

users/grfn/bbbg/src/bbbg/util/core.clj

@@ -52,6 +52,14 @@
   ([f] (map-kv f identity))
   ([f m] (map-kv f identity m)))
 
+(defn keep-keys
+  "Map f over the keys of m, keeping only those entries for which f does not
+  return nil. Preserves metadata on the incoming map. The one-argument form
+  returns a transducer that yields map-entries."
+  ([f] (keep (fn [[k v]] (when-let [k' (f k)]
+                          (first {k' v})))))
+  ([f m] (into (empty m) (keep-keys f) m)))
+
 (defn map-vals
   "Map f over the values of m. Preserves metadata on the incoming map. The
   one-argument form returns a transducer that yields map-entries."

users/grfn/bbbg/src/bbbg/util/spec.clj

@@ -0,0 +1,16 @@
+(ns bbbg.util.spec
+  (:require [expound.alpha :as exp]
+            [clojure.spec.alpha :as s]))
+
+(defn assert!
+  ([spec s] (assert! "Spec assertion failed" spec s))
+  ([message spec x]
+   (if (s/valid? spec x)
+     x
+     (throw (ex-info
+             (str message
+                  "\n"
+                  (exp/expound-str spec x))
+             (assoc (s/explain-data spec x)
+                    ::s/failure
+                    ::s/assertion-failed))))))

users/grfn/bbbg/test/bbbg/meetup/import_test.clj

@@ -0,0 +1,7 @@
+(ns bbbg.meetup.import-test
+  (:require [bbbg.meetup.import :as sut]
+            [clojure.test :refer :all]))
+
+(deftest test-row-user-id->user-id
+  (is (= "246364067" (sut/row-user-id->user-id "user 246364067")))
+  (is (= "246364067" (sut/row-user-id->user-id "246364067"))))