From 1aff49aa2e6f1ce54c42bcfe7939b7d372d27123 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 8 Jul 2024 15:03:48 -0600
Subject: [PATCH 001/183] [service] doc comment typo

---
 service/src/app.impl/observations/app.impl.observations.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/app.impl/observations/app.impl.observations.ts b/service/src/app.impl/observations/app.impl.observations.ts
index d2a184f36..158f398f0 100644
--- a/service/src/app.impl/observations/app.impl.observations.ts
+++ b/service/src/app.impl/observations/app.impl.observations.ts
@@ -163,7 +163,7 @@ export function registerDeleteRemovedAttachmentsHandler(domainEvents: EventEmitt
  * attachments should move to {@link Observation.assignTo()} so that method can
  * generate appropriate domain events, but that will require some API changes
  * in the entity layer, i.e., some alternative to the pre-generated ID
- * requirements for new form entries and attachments.  That could be soemthing
+ * requirements for new form entries and attachments.  That could be something
  * like generating pending identifiers that are easily distinguished from
  * persistence layer identifiers; maybe a `PendingId` class or `Id` class with
  * an `isPending` property.  That should be reasonable to implement, but no

From 0fbf8392933655fec8d5825933669136db3c88e8 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 8 Jul 2024 15:04:16 -0600
Subject: [PATCH 002/183] [service] line error on user routes

---
 service/src/adapters/users/adapters.users.controllers.web.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/adapters/users/adapters.users.controllers.web.ts b/service/src/adapters/users/adapters.users.controllers.web.ts
index 6b65cd435..11c053936 100644
--- a/service/src/adapters/users/adapters.users.controllers.web.ts
+++ b/service/src/adapters/users/adapters.users.controllers.web.ts
@@ -7,7 +7,7 @@ export interface UsersAppLayer {
   searchUsers: SearchUsers
 }
 
-export function UsersRoutes(app: UsersAppLayer, createAppRequest: WebAppRequestFactory) {
+export function UsersRoutes(app: UsersAppLayer, createAppRequest: WebAppRequestFactory): express.Router {
 
   const routes = express.Router()
 

From 8069148fb5fe53c1b109f1ff6205d0b09bc25204 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 8 Jul 2024 15:08:50 -0600
Subject: [PATCH 003/183] change postinstall script entry in root package to
 install:clean

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 4df4a9b00..bec258838 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
   "version": "6.3.0-beta.6",
   "files": [],
   "scripts": {
-    "postinstall": "npm-run-all service:ci web-app:ci image.service:ci nga-msi:ci",
+    "install:clean": "npm-run-all service:ci web-app:ci image.service:ci nga-msi:ci",
     "install:resolve": "npm-run-all service:install web-app:install image.service:install nga-msi:install",
     "build": "npm-run-all service:build web-app:build image.service:build nga-msi:build instance:build",
     "pack-all": "npm-run-all service:pack web-app:pack image.service:pack nga-msi:pack",

From 948d5cdaa82fcbd70cda33a3ca6d088825a5b847 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 10 Jul 2024 09:19:08 -0600
Subject: [PATCH 004/183] [service] users next: wip: move users schema elements
 from model to mongoose repository

---
 .../users/adapters.users.db.mongoose.ts       | 226 +++++++++++++++++-
 service/src/models/user.d.ts                  |  58 -----
 service/src/models/user.js                    | 163 -------------
 3 files changed, 222 insertions(+), 225 deletions(-)

diff --git a/service/src/adapters/users/adapters.users.db.mongoose.ts b/service/src/adapters/users/adapters.users.db.mongoose.ts
index 1e1eacd9e..ea1649f30 100644
--- a/service/src/adapters/users/adapters.users.db.mongoose.ts
+++ b/service/src/adapters/users/adapters.users.db.mongoose.ts
@@ -1,15 +1,233 @@
-import { User, UserId, UserRepository, UserFindParameters } from '../../entities/users/entities.users'
+import { User, UserId, UserRepository, UserFindParameters, UserIcon } from '../../entities/users/entities.users'
 import { BaseMongooseRepository, pageQuery } from '../base/adapters.base.db.mongoose'
 import { PageOf, pageOf } from '../../entities/entities.global'
-import * as legacy from '../../models/user'
 import _ from 'lodash'
 import mongoose from 'mongoose'
+import { RoleDocument, RoleJson } from '../../models/role'
+import { Authentication } from '../../entities/authentication/entities.authentication'
 
 export const UserModelName = 'User'
 
-export type UserDocument = legacy.UserDocument
+export type UserDocumentAttrs = Omit<User, 'authenticationId' | 'id' | 'roleId'> & {
+  _id: mongoose.Types.ObjectId,
+  authenticationId: mongoose.Types.ObjectId,
+  roleId: mongoose.Types.ObjectId,
+}
+
+export type UserDocument = mongoose.Document<mongoose.Types.ObjectId, any, UserDocumentAttrs> & {
+  // _id: mongoose.Types.ObjectId
+  // id: string
+  // username: string
+  // displayName: string
+  // email?: string
+  // phones: Phone[]
+  // avatar: Avatar
+  // icon: UserIcon
+  // active: boolean
+  // enabled: boolean
+  // roleId: mongoose.Types.ObjectId | RoleDocument
+  // authenticationId: mongoose.Types.ObjectId | mongoose.Document
+  // status?: string
+  // recentEventIds: number[]
+  // createdAt: Date
+  // lastUpdated: Date
+  // toJSON(): UserJson
+}
+
+// TODO: this probably needs an update now with new authentication changes
+export type UserJson = Omit<UserDocumentAttrs, '_id' | 'avatar' | 'roleId' | 'authenticationId' | keyof mongoose.Document>
+  & {
+    id: mongoose.Types.ObjectId,
+    icon: Omit<UserIcon, 'relativePath'>,
+    avatarUrl?: string,
+  }
+  & (RolePopulated | RoleReferenced)
+  & (AuthenticationPopulated | AuthenticationReferenced)
+
+type RoleReferenced = {
+  roleId: string,
+  role: never
+}
+
+type RolePopulated = {
+  roleId: never,
+  role: RoleJson
+}
+
+type AuthenticationPopulated = {
+  authenticationId: never,
+  authentication: Authentication
+}
+
+type AuthenticationReferenced = {
+  authenticationId: string,
+  authentication: never
+}
+
 export type UserModel = mongoose.Model<UserDocument>
-export const UserSchema = legacy.Model.schema
+
+export const UserPhoneSchema = new mongoose.Schema(
+  {
+    type: { type: String, required: true },
+    number: { type: String, required: true }
+  },
+  {
+    versionKey: false,
+    _id: false
+  }
+)
+
+export const UserSchema = new mongoose.Schema<UserDocumentAttrs>(
+  {
+    username: { type: String, required: true, unique: true },
+    displayName: { type: String, required: true },
+    email: { type: String, required: false },
+    phones: [ UserPhoneSchema ],
+    avatar: {
+      contentType: { type: String, required: false },
+      size: { type: Number, required: false },
+      relativePath: { type: String, required: false }
+    },
+    icon: {
+      type: { type: String, enum: ['none', 'upload', 'create'], default: 'none' },
+      text: { type: String },
+      color: { type: String },
+      contentType: { type: String, required: false },
+      size: { type: Number, required: false },
+      relativePath: { type: String, required: false }
+    },
+    active: { type: Boolean, required: true },
+    enabled: { type: Boolean, default: true, required: true },
+    roleId: { type: mongoose.Schema.Types.ObjectId, ref: 'Role', required: true },
+    status: { type: String, required: false, index: 'sparse' },
+    recentEventIds: [ { type: Number, ref: 'Event' } ],
+    authenticationId: { type: mongoose.Schema.Types.ObjectId, ref: 'Authentication', required: true }
+  },
+  {
+    versionKey: false,
+    timestamps: {
+      updatedAt: 'lastUpdated'
+    },
+    toObject: {
+      transform: DbUserToObject
+    },
+    toJSON: {
+      transform: DbUserToObject
+    }
+  }
+)
+
+UserSchema.virtual('authentication').get(function () {
+  return this.populated('authenticationId') ? this.authenticationId : null;
+});
+
+// Lowercase the username we store, this will allow for case insensitive usernames
+// Validate that username does not already exist
+UserSchema.pre('save', function (next) {
+  const user = this;
+  user.username = user.username.toLowerCase();
+  this.model('User').findOne({ username: user.username }, function (err, possibleDuplicate) {
+    if (err) return next(err);
+
+    if (possibleDuplicate && !possibleDuplicate._id.equals(user._id)) {
+      const error = new Error('username already exists');
+      error.status = 409;
+      return next(error);
+    }
+
+    next();
+  });
+});
+
+UserSchema.pre('save', function (next) {
+  const user = this;
+  if (user.active === false || user.enabled === false) {
+    Token.removeTokensForUser(user, function (err) {
+      next(err);
+    });
+  } else {
+    next();
+  }
+});
+
+UserSchema.pre('remove', function (next) {
+  const user = this;
+
+  async.parallel({
+    location: function (done) {
+      Location.removeLocationsForUser(user, done);
+    },
+    cappedlocation: function (done) {
+      CappedLocation.removeLocationsForUser(user, done);
+    },
+    token: function (done) {
+      Token.removeTokensForUser(user, done);
+    },
+    login: function (done) {
+      Login.removeLoginsForUser(user, done);
+    },
+    observation: function (done) {
+      Observation.removeUser(user, done);
+    },
+    eventAcl: function (done) {
+      Event.removeUserFromAllAcls(user, function (err) {
+        done(err);
+      });
+    },
+    teamAcl: function (done) {
+      Team.removeUserFromAllAcls(user, done);
+    },
+    authentication: function (done) {
+      Authentication.removeAuthenticationById(user.authenticationId, done);
+    }
+  },
+    function (err) {
+      next(err);
+    });
+});
+
+// eslint-disable-next-line complexity
+function DbUserToObject(user, userOut, options) {
+  userOut.id = userOut._id;
+  delete userOut._id;
+
+  delete userOut.avatar;
+
+  if (userOut.icon) { // TODO remove if check, icon is always there
+    delete userOut.icon.relativePath;
+  }
+
+  if (!!user.roleId && typeof user.roleId.toObject === 'function') {
+    userOut.role = user.roleId.toObject();
+    delete userOut.roleId;
+  }
+
+  /*
+  TODO: this used to use user.populated('authenticationId'), but when paging
+  and using the cursor(), mongoose was not setting the populated flag on the
+  cursor documents, so this condition was never met and paged user documents
+  erroneously retained the authenticationId key with the populated
+  authentication object.  this occurs in mage server 6.2.x with mongoose 4.x.
+  this might be fixed in mongoose 5+, so revisit on mage server 6.3.x.
+  */
+  if (!!user.authenticationId && typeof user.authenticationId.toObject === 'function') {
+    const authPlain = user.authenticationId.toObject({ virtuals: true });
+    delete userOut.authenticationId;
+    userOut.authentication = authPlain;
+  }
+
+  if (user.avatar && user.avatar.relativePath) {
+    // TODO, don't really like this, need a better way to set user resource, route
+    userOut.avatarUrl = [(options.path ? options.path : ""), "api", "users", user._id, "avatar"].join("/");
+  }
+
+  if (user.icon && user.icon.relativePath) {
+    // TODO, don't really like this, need a better way to set user resource, route
+    userOut.iconUrl = [(options.path ? options.path : ""), "api", "users", user._id, "icon"].join("/");
+  }
+
+  return userOut;
+};
 
 const idString = (x: mongoose.Document | mongoose.Types.ObjectId): string => {
   const id: mongoose.Types.ObjectId = x instanceof mongoose.Document ? x._id : x
diff --git a/service/src/models/user.d.ts b/service/src/models/user.d.ts
index 0ae581ae9..36c65f1d5 100644
--- a/service/src/models/user.d.ts
+++ b/service/src/models/user.d.ts
@@ -1,63 +1,5 @@
 import mongoose from 'mongoose'
-import { RoleJson, RoleDocument } from './role'
-import { UserIcon, Avatar, Phone } from '../entities/users/entities.users'
-import { Authentication } from '../entities/authentication/entities.authentication'
-
-
-export interface UserDocument extends mongoose.Document {
-  _id: mongoose.Types.ObjectId
-  id: string
-  username: string
-  displayName: string
-  email?: string
-  phones: Phone[]
-  avatar: Avatar
-  icon: UserIcon
-  active: boolean
-  enabled: boolean
-  roleId: mongoose.Types.ObjectId | RoleDocument
-  authenticationId: mongoose.Types.ObjectId | mongoose.Document
-  status?: string
-  recentEventIds: number[]
-  createdAt: Date
-  lastUpdated: Date
-  toJSON(): UserJson
-}
-
-
-// TODO: this probably needs an update now with new authentication changes
-export type UserJson = Omit<UserDocument, '_id' | 'avatar' | 'roleId' | 'authenticationId' | keyof mongoose.Document>
-  & {
-    id: mongoose.Types.ObjectId,
-    icon: Omit<UserIcon, 'relativePath'>,
-    avatarUrl?: string,
-  }
-  & (RolePopulated | RoleReferenced)
-  & (AuthenticationPopulated | AuthenticationReferenced)
 
 export declare const Model: mongoose.Model<UserDocument>
-
 export function getUserById(id: mongoose.Types.ObjectId): Promise<UserDocument | null>
 export function getUserById(id: mongoose.Types.ObjectId, callback: (err: null | any, result: UserDocument | null) => any): void
-
-type RoleReferenced = {
-  roleId: string,
-  role: never
-}
-
-type RolePopulated = {
-  roleId: never,
-  role: RoleJson
-}
-
-type AuthenticationPopulated = {
-  authenticationId: never,
-  authentication: Authentication
-}
-
-type AuthenticationReferenced = {
-  authenticationId: string,
-  authentication: never
-}
-
-
diff --git a/service/src/models/user.js b/service/src/models/user.js
index f31225dff..554561e8a 100644
--- a/service/src/models/user.js
+++ b/service/src/models/user.js
@@ -16,169 +16,6 @@ const mongoose = require('mongoose')
   , { pageOf } = require('../entities/entities.global')
   , FilterParser = require('../utilities/filterParser');
 
-// Creates a new Mongoose Schema object
-const Schema = mongoose.Schema;
-
-const PhoneSchema = new Schema({
-  type: { type: String, required: true },
-  number: { type: String, required: true }
-}, {
-  versionKey: false,
-  _id: false
-});
-
-// Collection to hold users
-const UserSchema = new Schema(
-  {
-    username: { type: String, required: true, unique: true },
-    displayName: { type: String, required: true },
-    email: { type: String, required: false },
-    phones: [PhoneSchema],
-    avatar: {
-      contentType: { type: String, required: false },
-      size: { type: Number, required: false },
-      relativePath: { type: String, required: false }
-    },
-    icon: {
-      type: { type: String, enum: ['none', 'upload', 'create'], default: 'none' },
-      text: { type: String },
-      color: { type: String },
-      contentType: { type: String, required: false },
-      size: { type: Number, required: false },
-      relativePath: { type: String, required: false }
-    },
-    active: { type: Boolean, required: true },
-    enabled: { type: Boolean, default: true, required: true },
-    roleId: { type: Schema.Types.ObjectId, ref: 'Role', required: true },
-    status: { type: String, required: false, index: 'sparse' },
-    recentEventIds: [{ type: Number, ref: 'Event' }],
-    authenticationId: { type: Schema.Types.ObjectId, ref: 'Authentication', required: true }
-  },
-  {
-    versionKey: false,
-    timestamps: {
-      updatedAt: 'lastUpdated'
-    },
-    toObject: {
-      transform: DbUserToObject
-    },
-    toJSON: {
-      transform: DbUserToObject
-    }
-  }
-);
-
-UserSchema.virtual('authentication').get(function () {
-  return this.populated('authenticationId') ? this.authenticationId : null;
-});
-
-// Lowercase the username we store, this will allow for case insensitive usernames
-// Validate that username does not already exist
-UserSchema.pre('save', function (next) {
-  const user = this;
-  user.username = user.username.toLowerCase();
-  this.model('User').findOne({ username: user.username }, function (err, possibleDuplicate) {
-    if (err) return next(err);
-
-    if (possibleDuplicate && !possibleDuplicate._id.equals(user._id)) {
-      const error = new Error('username already exists');
-      error.status = 409;
-      return next(error);
-    }
-
-    next();
-  });
-});
-
-UserSchema.pre('save', function (next) {
-  const user = this;
-  if (user.active === false || user.enabled === false) {
-    Token.removeTokensForUser(user, function (err) {
-      next(err);
-    });
-  } else {
-    next();
-  }
-});
-
-UserSchema.pre('remove', function (next) {
-  const user = this;
-
-  async.parallel({
-    location: function (done) {
-      Location.removeLocationsForUser(user, done);
-    },
-    cappedlocation: function (done) {
-      CappedLocation.removeLocationsForUser(user, done);
-    },
-    token: function (done) {
-      Token.removeTokensForUser(user, done);
-    },
-    login: function (done) {
-      Login.removeLoginsForUser(user, done);
-    },
-    observation: function (done) {
-      Observation.removeUser(user, done);
-    },
-    eventAcl: function (done) {
-      Event.removeUserFromAllAcls(user, function (err) {
-        done(err);
-      });
-    },
-    teamAcl: function (done) {
-      Team.removeUserFromAllAcls(user, done);
-    },
-    authentication: function (done) {
-      Authentication.removeAuthenticationById(user.authenticationId, done);
-    }
-  },
-    function (err) {
-      next(err);
-    });
-});
-
-// eslint-disable-next-line complexity
-function DbUserToObject(user, userOut, options) {
-  userOut.id = userOut._id;
-  delete userOut._id;
-
-  delete userOut.avatar;
-
-  if (userOut.icon) { // TODO remove if check, icon is always there
-    delete userOut.icon.relativePath;
-  }
-
-  if (!!user.roleId && typeof user.roleId.toObject === 'function') {
-    userOut.role = user.roleId.toObject();
-    delete userOut.roleId;
-  }
-
-  /*
-  TODO: this used to use user.populated('authenticationId'), but when paging
-  and using the cursor(), mongoose was not setting the populated flag on the
-  cursor documents, so this condition was never met and paged user documents
-  erroneously retained the authenticationId key with the populated
-  authentication object.  this occurs in mage server 6.2.x with mongoose 4.x.
-  this might be fixed in mongoose 5+, so revisit on mage server 6.3.x.
-  */
-  if (!!user.authenticationId && typeof user.authenticationId.toObject === 'function') {
-    const authPlain = user.authenticationId.toObject({ virtuals: true });
-    delete userOut.authenticationId;
-    userOut.authentication = authPlain;
-  }
-
-  if (user.avatar && user.avatar.relativePath) {
-    // TODO, don't really like this, need a better way to set user resource, route
-    userOut.avatarUrl = [(options.path ? options.path : ""), "api", "users", user._id, "avatar"].join("/");
-  }
-
-  if (user.icon && user.icon.relativePath) {
-    // TODO, don't really like this, need a better way to set user resource, route
-    userOut.iconUrl = [(options.path ? options.path : ""), "api", "users", user._id, "icon"].join("/");
-  }
-
-  return userOut;
-};
 
 exports.transform = DbUserToObject;
 

From 035b21d4c99646e02d0977fae5a7f87cba5408d4 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 10 Jul 2024 13:35:04 -0600
Subject: [PATCH 005/183] [service] users next: remove `status` field from
 users schema and related api routes, which does not seem to be used by any
 client or anywhere else in the service

---
 .../users/adapters.users.db.mongoose.ts       |  1 -
 service/src/docs/openapi.yaml                 | 32 -------------------
 service/src/models/user.js                    |  7 ----
 service/src/routes/users.js                   | 29 -----------------
 4 files changed, 69 deletions(-)

diff --git a/service/src/adapters/users/adapters.users.db.mongoose.ts b/service/src/adapters/users/adapters.users.db.mongoose.ts
index ea1649f30..e29d75b8f 100644
--- a/service/src/adapters/users/adapters.users.db.mongoose.ts
+++ b/service/src/adapters/users/adapters.users.db.mongoose.ts
@@ -99,7 +99,6 @@ export const UserSchema = new mongoose.Schema<UserDocumentAttrs>(
     active: { type: Boolean, required: true },
     enabled: { type: Boolean, default: true, required: true },
     roleId: { type: mongoose.Schema.Types.ObjectId, ref: 'Role', required: true },
-    status: { type: String, required: false, index: 'sparse' },
     recentEventIds: [ { type: Number, ref: 'Event' } ],
     authenticationId: { type: mongoose.Schema.Types.ObjectId, ref: 'Authentication', required: true }
   },
diff --git a/service/src/docs/openapi.yaml b/service/src/docs/openapi.yaml
index 8da5c9b1c..e91abc3aa 100644
--- a/service/src/docs/openapi.yaml
+++ b/service/src/docs/openapi.yaml
@@ -461,36 +461,6 @@ paths:
           content:
             application/json:
               schema: { $ref: '#/components/schemas/User' }
-  /api/users/myself/status:
-    put:
-      tags: [ User ]
-      description: Update the status of the requesting user.
-      operationId: updateMyStatus
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                status:
-                  type: string
-              required: [ status ]
-      responses:
-        200:
-          description: Successful status update
-          content:
-            application/json:
-              schema: { $ref: '#/components/schemas/User' }
-    delete:
-      tags: [ User ]
-      description: Delete the status of the requesting user.
-      operationId: deleteMyStatus
-      responses:
-        200:
-          description: Successfully deleted status
-          content:
-            application/json:
-              schema: { $ref: '#/components/schemas/User' }
   /api/users/{userId}:
     parameters:
       - { $ref: '#/components/parameters/userIdInPath' }
@@ -3064,8 +3034,6 @@ components:
           type: string
         displayName:
           type: string
-        status:
-          type: string
         email:
           type: string
           format: email
diff --git a/service/src/models/user.js b/service/src/models/user.js
index 554561e8a..5d3645a77 100644
--- a/service/src/models/user.js
+++ b/service/src/models/user.js
@@ -218,13 +218,6 @@ exports.validLogin = async function (user) {
   await user.authentication.save();
 };
 
-exports.setStatusForUser = function (user, status, callback) {
-  const update = { status: status };
-  User.findByIdAndUpdate(user._id, update, { new: true }, function (err, user) {
-    callback(err, user);
-  });
-};
-
 exports.setRoleForUser = function (user, role, callback) {
   const update = { role: role };
   User.findByIdAndUpdate(user._id, update, { new: true }, function (err, user) {
diff --git a/service/src/routes/users.js b/service/src/routes/users.js
index 4c7cf4b0d..82785e24a 100644
--- a/service/src/routes/users.js
+++ b/service/src/routes/users.js
@@ -419,35 +419,6 @@ module.exports = function (app, security) {
     }
   );
 
-  // update status for myself
-  app.put(
-    '/api/users/myself/status',
-    passport.authenticate('bearer'),
-    function (req, res) {
-      var status = req.param('status');
-      if (!status) return res.status(400).send("Missing required parameter 'status'");
-      req.user.status = status;
-
-      new api.User().update(req.user, function (err, updatedUser) {
-        updatedUser = userTransformer.transform(updatedUser, { path: req.getRoot() });
-        res.json(updatedUser);
-      });
-    }
-  );
-
-  // remove status for myself
-  app.delete(
-    '/api/users/myself/status',
-    passport.authenticate('bearer'),
-    function (req, res) {
-      req.user.status = undefined;
-      new api.User().update(req.user, function (err, updatedUser) {
-        updatedUser = userTransformer.transform(updatedUser, { path: req.getRoot() });
-        res.json(updatedUser);
-      });
-    }
-  );
-
   // get user by id
   app.get(
     '/api/users/:userId',

From cd90692d0416672d27cd121304848329fc8a4714 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 10 Jul 2024 22:21:38 -0600
Subject: [PATCH 006/183] [service] users next: add missing properties to user
 entity

---
 service/src/entities/users/entities.users.ts | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/service/src/entities/users/entities.users.ts b/service/src/entities/users/entities.users.ts
index 93cd0b25b..9e90eaa24 100644
--- a/service/src/entities/users/entities.users.ts
+++ b/service/src/entities/users/entities.users.ts
@@ -1,6 +1,7 @@
 import { PagingParameters, PageOf } from '../entities.global'
 import { Role } from '../authorization/entities.authorization'
 import { Authentication } from '../authentication/entities.authentication'
+import { MageEventId } from '../events/entities.events'
 
 export type UserId = string
 
@@ -16,6 +17,14 @@ export interface User {
   phones: Phone[]
   roleId: string
   authenticationId: string
+  avatar: Avatar
+  icon: UserIcon
+  /**
+   * TODO: this could move to another entity like `UserExperience` or
+   * `UserSettings` to eliminate the cyclic reference between the user and
+   * event modules.
+   */
+  recentEventIds: MageEventId[]
   // TODO: the rest of the properties
 }
 

From b2c5bfc5293f7b6465c99e510f3b660f11c78011 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 10 Jul 2024 22:22:04 -0600
Subject: [PATCH 007/183] [service] users next: remove unused import

---
 service/src/adapters/events/adapters.events.db.mongoose.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/service/src/adapters/events/adapters.events.db.mongoose.ts b/service/src/adapters/events/adapters.events.db.mongoose.ts
index ba37e545f..4027628e1 100644
--- a/service/src/adapters/events/adapters.events.db.mongoose.ts
+++ b/service/src/adapters/events/adapters.events.db.mongoose.ts
@@ -3,7 +3,6 @@ import { MageEventRepository, MageEventAttrs, MageEventId, MageEvent } from '../
 import mongoose from 'mongoose'
 import { FeedId } from '../../entities/feeds/entities.feeds'
 import * as legacy from '../../models/event'
-import { Team } from '../../entities/teams/entities.teams'
 
 export const MageEventModelName = 'Event'
 

From 5b7e2c41f50ddb1792b9b9a6e5f75b280fbc53f2 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 10 Jul 2024 22:22:42 -0600
Subject: [PATCH 008/183] [service] users next: add comments with todo notes

---
 service/src/entities/teams/entities.teams.ts | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/service/src/entities/teams/entities.teams.ts b/service/src/entities/teams/entities.teams.ts
index 08cabdf5f..df874cdf3 100644
--- a/service/src/entities/teams/entities.teams.ts
+++ b/service/src/entities/teams/entities.teams.ts
@@ -14,10 +14,17 @@ export interface Team {
    * that MAGE creates for each event.  When an event manager or administrator
    * adds participant users to an event individually, as opposed to an entire
    * team, MAGE places the users in the event's _event team_.
+   *
+   * TODO: this should perhaps be an encapsulated detail of the data layer,
+   * i.e., the notion that an event has an implicit team to manage direct/adhoc
+   * user membership.
    */
   teamEventId?: MageEventId
 }
 
+/**
+ * TODO: move to permission/authorization module
+ */
 export interface TeamAcl {
   [userId: string]: {
     role: TeamMemberRole,

From bc40e151de161b35e4aa76183fbf8a0dd8908d2e Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 11 Jul 2024 19:10:49 -0600
Subject: [PATCH 009/183] [service] users next: add comments with todo notes

---
 service/src/access/index.ts                    |  1 +
 service/src/api/event.js                       |  1 +
 service/src/api/user.js                        |  2 ++
 .../app.impl/systemInfo/app.impl.systemInfo.ts | 18 +++++++++---------
 service/src/app.ts                             |  1 +
 service/src/authentication/index.js            | 11 +++++++----
 service/src/authentication/ldap.js             |  3 ++-
 service/src/authentication/local.js            |  3 +++
 service/src/authentication/oauth.js            |  3 ++-
 service/src/authentication/openidconnect.js    |  3 ++-
 service/src/authentication/saml.js             |  4 +++-
 service/src/export/csv.ts                      |  3 +++
 service/src/export/geojson.ts                  |  1 +
 service/src/export/geopackage.ts               |  1 +
 service/src/export/kml.ts                      |  1 +
 service/src/export/kmlWriter.ts                |  1 +
 service/src/migrations/007-user-icon.js        |  1 +
 service/src/models/authentication.js           |  1 +
 service/src/models/device.js                   |  2 ++
 service/src/models/event.js                    |  2 ++
 service/src/models/export.d.ts                 |  1 +
 service/src/models/role.js                     |  2 +-
 service/src/models/team.d.ts                   |  1 +
 service/src/models/team.js                     |  2 ++
 service/src/permissions/permissions.events.ts  |  1 +
 .../permissions/permissions.role-based.base.ts |  1 +
 service/src/routes/index.js                    |  1 +
 service/src/routes/observations.js             |  2 +-
 service/src/routes/setup.js                    |  2 ++
 29 files changed, 57 insertions(+), 19 deletions(-)

diff --git a/service/src/access/index.ts b/service/src/access/index.ts
index 4b1934c33..2f660cc71 100644
--- a/service/src/access/index.ts
+++ b/service/src/access/index.ts
@@ -36,6 +36,7 @@ export = Object.freeze({
     }
   },
 
+  // TODO: users-next
   userHasPermission(user: UserDocument, permission: AnyPermission) {
     if (!user || !user.roleId) {
       return false
diff --git a/service/src/api/event.js b/service/src/api/event.js
index 70a067878..87951a753 100644
--- a/service/src/api/event.js
+++ b/service/src/api/event.js
@@ -130,6 +130,7 @@ Event.prototype.deleteEvent = async function(callback) {
         });
       },
       recentEventIds: function(done) {
+        // TODO: users-next
         User.removeRecentEventForUsers(event, function(err) {
           done(err);
         });
diff --git a/service/src/api/user.js b/service/src/api/user.js
index e58e8bb3c..d13407939 100644
--- a/service/src/api/user.js
+++ b/service/src/api/user.js
@@ -11,6 +11,8 @@ const UserModel = require('../models/user')
   , async = require('async')
   , environment = require('../environment/env');
 
+// TODO: users-next
+
 const userBase = environment.userBaseDirectory;
 
 function contentPath(id, user, content, type) {
diff --git a/service/src/app.impl/systemInfo/app.impl.systemInfo.ts b/service/src/app.impl/systemInfo/app.impl.systemInfo.ts
index 23bf80d4d..4054701c8 100644
--- a/service/src/app.impl/systemInfo/app.impl.systemInfo.ts
+++ b/service/src/app.impl/systemInfo/app.impl.systemInfo.ts
@@ -54,19 +54,19 @@ export function CreateReadSystemInfo(
   req: api.ReadSystemInfoRequest
   ): Promise<api.ReadSystemInfoResponse> {
     const isAuthenticated = req.context.requestingPrincipal() != null;
-
-	// FIXME: Replace this with Robert's first-run secret implementation when available
-	const legacyUsers = Users as any;   
+    // FIXME: Replace this with Robert's first-run secret implementation when available
+    // TODO: users-next
+    const legacyUsers = Users as any;
     const userCount = await new Promise(resolve => {
       legacyUsers.count({}, (err:any, count:any) => {
-	    resolve(count)
-	  });
+        resolve(count)
+      });
     });
-    
+
     // Initialize with base system info
-    let systemInfoResponse: ExoRedactedSystemInfo = {	  
-      version: versionInfo,	  
-	  initial: userCount == 0,
+    let systemInfoResponse: ExoRedactedSystemInfo = {
+      version: versionInfo,
+      initial: userCount == 0,
       disclaimer: (await settingsModule.getSetting('disclaimer')) || {},
       contactInfo: (await settingsModule.getSetting('contactInfo')) || {}
     };
diff --git a/service/src/app.ts b/service/src/app.ts
index c7faab2aa..f5af7a8c3 100644
--- a/service/src/app.ts
+++ b/service/src/app.ts
@@ -20,6 +20,7 @@ import { PreFetchedUserRoleFeedsPermissionService } from './permissions/permissi
 import { FeedsRoutes } from './adapters/feeds/adapters.feeds.controllers.web'
 import { WebAppRequestFactory } from './adapters/adapters.controllers.web'
 import { AppRequest, AppRequestContext } from './app.api/app.api.global'
+// TODO: users-next
 import { UserDocument } from './models/user'
 import SimpleIdFactory from './adapters/adapters.simple_id_factory'
 import { JsonSchemaService, JsonValidator, JSONSchema4 } from './entities/entities.json_types'
diff --git a/service/src/authentication/index.js b/service/src/authentication/index.js
index d7bd03611..7335061b7 100644
--- a/service/src/authentication/index.js
+++ b/service/src/authentication/index.js
@@ -22,15 +22,17 @@ class AuthenticationInitializer {
     AuthenticationInitializer.passport = passport;
     AuthenticationInitializer.provision = provision;
 
-    const BearerStrategy = require('passport-http-bearer').Strategy
-      , User = require('../models/user')
-      , Token = require('../models/token');
+    const BearerStrategy = require('passport-http-bearer').Strategy;
+    // TODO: users-next
+    const User = require('../models/user');
+    const Token = require('../models/token');
 
     passport.serializeUser(function (user, done) {
       done(null, user._id);
     });
 
     passport.deserializeUser(function (id, done) {
+      // TODO: users-next
       User.getUserById(id, function (err, user) {
         done(err, user);
       });
@@ -64,6 +66,7 @@ class AuthenticationInitializer {
 
       AuthenticationInitializer.tokenService.verifyToken(token, expectation)
         .then(payload => {
+          // TODO: users-next
           User.getUserById(payload.subject)
             .then(user => done(null, user))
             .catch(err => done(err));
@@ -92,7 +95,7 @@ class AuthenticationInitializer {
           userAgent: req.headers['user-agent'],
           appVersion: req.param('appVersion')
         };
-
+        // TODO: users-next
         new api.User().login(req.user, req.provisionedDevice, options, function (err, token) {
           if (err) return next(err);
 
diff --git a/service/src/authentication/ldap.js b/service/src/authentication/ldap.js
index 719808363..279dca8f4 100644
--- a/service/src/authentication/ldap.js
+++ b/service/src/authentication/ldap.js
@@ -27,6 +27,7 @@ function configure(strategy) {
   },
     function (profile, done) {
       const username = profile[strategy.settings.profile.id ];
+      // TODO: users-next
       User.getUserByAuthenticationStrategy(strategy.type, username, function (err, user) {
         if (err) return done(err);
 
@@ -49,7 +50,7 @@ function configure(strategy) {
                 }
               }
             };
-
+            // TODO: users-next
             new api.User().create(user).then(newUser => {
               if (!newUser.authentication.authenticationConfiguration.enabled) {
                 log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
diff --git a/service/src/authentication/local.js b/service/src/authentication/local.js
index f438419ad..5193d960f 100644
--- a/service/src/authentication/local.js
+++ b/service/src/authentication/local.js
@@ -11,6 +11,7 @@ function configure() {
   log.info('Configuring local authentication');
   passport.use(new LocalStrategy(
     function (username, password, done) {
+      // TODO: users-next
       User.getUserByUsername(username, function (err, user) {
         if (err) { return done(err); }
 
@@ -49,11 +50,13 @@ function configure() {
           if (err) return done(err);
 
           if (isValid) {
+            // TODO: users-next
             User.validLogin(user)
               .then(() => done(null, user))
               .catch(err => done(err));
           } else {
             log.warn('Failed login attempt: User with username ' + username + ' provided an invalid password');
+            // TODO: users-next
             User.invalidLogin(user)
               .then(() => done(null, false, { message: 'Please check your username and password and try again.' }))
               .catch(err => done(err));
diff --git a/service/src/authentication/oauth.js b/service/src/authentication/oauth.js
index a531c69ed..63ce786fb 100644
--- a/service/src/authentication/oauth.js
+++ b/service/src/authentication/oauth.js
@@ -73,6 +73,7 @@ function configure(strategy) {
 
       const profileId = profile[strategy.settings.profile.id];
 
+      // TODO: users-next
       User.getUserByAuthenticationStrategy(strategy.type, profileId, function (err, user) {
          if (err) return done(err);
 
@@ -109,7 +110,7 @@ function configure(strategy) {
                      }
                   }
                };
-
+               // TODO: users-next
                new api.User().create(user).then(newUser => {
                   if (!newUser.authentication.authenticationConfiguration.enabled) {
                      log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
diff --git a/service/src/authentication/openidconnect.js b/service/src/authentication/openidconnect.js
index 88cfc9e1b..979026786 100644
--- a/service/src/authentication/openidconnect.js
+++ b/service/src/authentication/openidconnect.js
@@ -26,6 +26,7 @@ function configure(strategy) {
       return done(`OIDC user profile does not contain id property ${strategy.settings.profile.id}`);
     }
 
+    // TODO: users-next
     User.getUserByAuthenticationStrategy(strategy.type, profileId, function (err, user) {
       if (err) return done(err);
 
@@ -48,7 +49,7 @@ function configure(strategy) {
               }
             }
           };
-
+          // TODO: users-next
           new api.User().create(user).then(newUser => {
             if (!newUser.authentication.authenticationConfiguration.enabled) {
               log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
diff --git a/service/src/authentication/saml.js b/service/src/authentication/saml.js
index 0c1a5ffaa..85ab11373 100644
--- a/service/src/authentication/saml.js
+++ b/service/src/authentication/saml.js
@@ -81,6 +81,7 @@ function configure(strategy) {
       return done('Failed to load user id from SAML profile');
     }
 
+    // TODO: users-next
     User.getUserByAuthenticationStrategy(strategy.type, uid, function (err, user) {
       if (err) return done(err);
 
@@ -103,7 +104,7 @@ function configure(strategy) {
               }
             }
           };
-
+          // TODO: users-next
           new api.User().create(user).then(newUser => {
             if (!newUser.authentication.authenticationConfiguration.enabled) {
               log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
@@ -304,6 +305,7 @@ function initialize(strategy) {
     provision.check(strategy.name),
     parseLoginMetadata,
     function (req, res, next) {
+      // TODO: users-next
       new api.User().login(req.user, req.provisionedDevice, req.loginOptions, function (err, token) {
         if (err) return next(err);
 
diff --git a/service/src/export/csv.ts b/service/src/export/csv.ts
index 8df82aebd..d255f539c 100644
--- a/service/src/export/csv.ts
+++ b/service/src/export/csv.ts
@@ -143,8 +143,10 @@ export class Csv extends Exporter {
       ...observation.properties
     } as any
 
+    // TODO: users-next
     if (!cache.user || cache.user._id.toString() !== observation.userId?.toString()) {
       if (observation.userId) {
+        // TODO: users-next
         cache.user = await User.getUserById(observation.userId!)
       }
     }
@@ -243,6 +245,7 @@ export class Csv extends Exporter {
       mgrs: mgrs.forward(location.geometry.coordinates),
     } as UserLocationDocumentProperties & { user?: string, device?: string }
     if (!cache.user || cache.user._id.toString() !== location.userId.toString()) {
+      // TODO: users-next
       cache.user = await User.getUserById(location.userId)
     }
     if (!cache.device || cache.device._id.toString() !== flat.deviceId?.toString()) {
diff --git a/service/src/export/geojson.ts b/service/src/export/geojson.ts
index 2dd4c5a84..47626c2c2 100755
--- a/service/src/export/geojson.ts
+++ b/service/src/export/geojson.ts
@@ -127,6 +127,7 @@ export class GeoJson extends Exporter {
       this.mapObservationProperties(observation, archive);
 
       if (observation.userId) {
+        // TODO: users-next
         if (!user || user._id.toString() !== String(observation.userId)) {
           user = await User.getUserById(observation.userId)
         }
diff --git a/service/src/export/geopackage.ts b/service/src/export/geopackage.ts
index 0a20eef67..cae3bffb6 100644
--- a/service/src/export/geopackage.ts
+++ b/service/src/export/geopackage.ts
@@ -205,6 +205,7 @@ export class GeoPackage extends Exporter {
     const userIconRows: Map<string, IconRow> = new Map()
     let zoomToEnvelope: Envelope | null = null
     return cursor.eachAsync(async location => {
+      // TODO: users-next
       if (user?._id.toString() !== location.userId.toString()) {
         user = await User.getUserById(location.userId);
       }
diff --git a/service/src/export/kml.ts b/service/src/export/kml.ts
index ec04198db..ac1329e7d 100755
--- a/service/src/export/kml.ts
+++ b/service/src/export/kml.ts
@@ -107,6 +107,7 @@ export class Kml extends Exporter {
         }
         locationString = '';
         lastUserId = location.userId.toString();
+        // TODO: users-next
         lastUser = await User.getUserById(location.userId);
         if (lastUser) {
           userStyles += writer.generateUserStyle(lastUser);
diff --git a/service/src/export/kmlWriter.ts b/service/src/export/kmlWriter.ts
index f1e6559b4..5ad8383e1 100755
--- a/service/src/export/kmlWriter.ts
+++ b/service/src/export/kmlWriter.ts
@@ -100,6 +100,7 @@ export function generateKMLFolderStart(name: string): string {
   return `<Folder><name>${name}</name>`;
 }
 
+// TODO: users-next
 export function generateUserStyle(user: UserDocument): string {
   if (user.icon && user.icon.relativePath) {
     return fragment({
diff --git a/service/src/migrations/007-user-icon.js b/service/src/migrations/007-user-icon.js
index 65933da3e..dea8b81a3 100644
--- a/service/src/migrations/007-user-icon.js
+++ b/service/src/migrations/007-user-icon.js
@@ -6,6 +6,7 @@ exports.id = '007-user-icon';
 exports.up = function(done) {
   this.log('updating user icons');
 
+  // TODO: users-next
   User.getUsers(function(err, users) {
     if (err) return done(err);
 
diff --git a/service/src/models/authentication.js b/service/src/models/authentication.js
index 86505e2f9..c30288121 100644
--- a/service/src/models/authentication.js
+++ b/service/src/models/authentication.js
@@ -124,6 +124,7 @@ LocalSchema.pre('save', function (next) {
 
   async.waterfall([
     function (done) {
+      // TODO: users-next
       User.getUserByAuthenticationId(authentication._id, function (err, user) {
         done(err, user);
       });
diff --git a/service/src/models/device.js b/service/src/models/device.js
index 34891b4bf..9a2e1d6fa 100644
--- a/service/src/models/device.js
+++ b/service/src/models/device.js
@@ -29,6 +29,7 @@ DeviceSchema.path('userId').validate(async function (userId) {
   let isValid = true;
 
   try {
+    // TODO: users-next
     const user = await User.getUserById(userId);
     if (!user) {
       isValid = false;
@@ -181,6 +182,7 @@ async function queryUsersAndDevicesThenPage(options, conditions) {
     registered = conditions.registered;
     delete conditions.registered;
   }
+  // TODO: users-next
   const count = await User.Model.count(conditions);
   return User.Model.find(conditions, "_id").populate({ path: 'authenticationId', populate: { path: 'authenticationConfigurationId' } }).exec().then(data => {
     const ids = [];
diff --git a/service/src/models/event.js b/service/src/models/event.js
index 08e5a8d19..1f0e8c804 100644
--- a/service/src/models/event.js
+++ b/service/src/models/event.js
@@ -593,6 +593,7 @@ exports.getMembers = async function (eventId, options) {
 
     // per https://docs.mongodb.com/v5.0/reference/method/cursor.sort/#sort-consistency,
     // add _id to sort to ensure consistent ordering
+    // TODO: users-next
     const members = await User.Model.find(params)
       .sort('displayName _id')
       .limit(options.pageSize)
@@ -606,6 +607,7 @@ exports.getMembers = async function (eventId, options) {
 
     const includeTotalCount = typeof options.includeTotalCount === 'boolean' ? options.includeTotalCount : options.pageIndex === 0
     if (includeTotalCount) {
+      // TODO: users-next
       page.totalCount = await User.Model.count(params);
     }
 
diff --git a/service/src/models/export.d.ts b/service/src/models/export.d.ts
index ff0fe9c28..0fbce07a5 100644
--- a/service/src/models/export.d.ts
+++ b/service/src/models/export.d.ts
@@ -36,6 +36,7 @@ export type ExportAttrs = {
 export type ExportDocument = ExportAttrs & mongoose.Document
 
 export type ExportDocumentPopulated = Omit<ExportDocument, 'userId' | 'options'> & {
+  // TODO: users-next
   userId: UserDocument | null,
   options: Omit<ExportOptions, 'eventId'> & {
     event: { _id: number, name: string }
diff --git a/service/src/models/role.js b/service/src/models/role.js
index 7d1a5df10..406ceb080 100644
--- a/service/src/models/role.js
+++ b/service/src/models/role.js
@@ -21,7 +21,7 @@ const RoleSchema = new Schema(
 
 RoleSchema.pre('remove', function (next) {
   const role = this;
-
+  // TODO: users-next
   User.removeRoleFromUsers(role, function (err) {
     next(err);
   });
diff --git a/service/src/models/team.d.ts b/service/src/models/team.d.ts
index 71a055d01..d8c2f3c77 100644
--- a/service/src/models/team.d.ts
+++ b/service/src/models/team.d.ts
@@ -28,4 +28,5 @@ export function removeUserFromAclForEventTeam(eventId: any, userId: any, callbac
 export function removeUserFromAllAcls(user: any, callback: any): void;
 declare var Team: mongoose.Model<mongoose.Document>;
 import mongoose = require("mongoose");
+// TODO: users-next
 import User from './user'
\ No newline at end of file
diff --git a/service/src/models/team.js b/service/src/models/team.js
index 72cfedcef..c15cbde94 100644
--- a/service/src/models/team.js
+++ b/service/src/models/team.js
@@ -186,6 +186,7 @@ exports.getMembers = async function (teamId, options) {
 
     // per https://docs.mongodb.com/v5.0/reference/method/cursor.sort/#sort-consistency,
     // add _id to sort to ensure consistent ordering
+    // TODO: users-next
     const members = await User.Model.find(params)
       .sort('displayName _id')
       .limit(options.pageSize)
@@ -199,6 +200,7 @@ exports.getMembers = async function (teamId, options) {
 
     const includeTotalCount = typeof options.includeTotalCount === 'boolean' ? options.includeTotalCount : options.pageIndex === 0
     if (includeTotalCount) {
+      // TODO: users-next
       page.totalCount = await User.Model.count(params);
     }
 
diff --git a/service/src/permissions/permissions.events.ts b/service/src/permissions/permissions.events.ts
index 899d4c952..f03e95cc6 100644
--- a/service/src/permissions/permissions.events.ts
+++ b/service/src/permissions/permissions.events.ts
@@ -12,6 +12,7 @@ import { UserId } from '../entities/users/entities.users'
 import { MongooseMageEventRepository } from '../adapters/events/adapters.events.db.mongoose'
 import { TeamId } from '../entities/teams/entities.teams'
 
+// TODO: users-next
 export interface EventRequestContext extends AppRequestContext<UserDocument> {
   readonly event: MageEventAttrs | MageEventDocument
 }
diff --git a/service/src/permissions/permissions.role-based.base.ts b/service/src/permissions/permissions.role-based.base.ts
index 57c545ef1..a51ccfe08 100644
--- a/service/src/permissions/permissions.role-based.base.ts
+++ b/service/src/permissions/permissions.role-based.base.ts
@@ -8,6 +8,7 @@ import { AnyPermission } from '../entities/authorization/entities.permissions'
  * TODO: This should not be statically linked to the Mongoose Document type but
  * for now this is the quick and dirty way because the legacy web adapter layer
  * puts the user Mongoose document on the request.
+ * TODO: users-next
  */
 export type UserWithRole = UserDocument & {
   roleId: RoleDocument
diff --git a/service/src/routes/index.js b/service/src/routes/index.js
index 12ad25196..2787b0b8b 100644
--- a/service/src/routes/index.js
+++ b/service/src/routes/index.js
@@ -33,6 +33,7 @@ module.exports = function(app, security) {
     if (!/^[0-9a-f]{24}$/.test(userId)) {
       return res.status(400).send('Invalid user ID in request path');
     }
+    // TODO: users-next
     new api.User().getById(userId, function(err, user) {
       if (!user) return res.status(404).send('User not found');
       req.userParam = user;
diff --git a/service/src/routes/observations.js b/service/src/routes/observations.js
index a3496f64a..d27ed60e3 100644
--- a/service/src/routes/observations.js
+++ b/service/src/routes/observations.js
@@ -142,7 +142,7 @@ module.exports = function (app, security) {
   function getUserForObservation(req, res, next) {
     var userId = req.observation.userId;
     if (!userId) return next();
-
+    // TODO: users-next
     new api.User().getById(userId, function (err, user) {
       if (err) return next(err);
 
diff --git a/service/src/routes/setup.js b/service/src/routes/setup.js
index 5dc5ad2ae..ae2462f72 100644
--- a/service/src/routes/setup.js
+++ b/service/src/routes/setup.js
@@ -7,6 +7,7 @@ module.exports = function (app, security) {
     , AuthenticationConfiguration = require('../models/authenticationconfiguration');
 
   function authorizeSetup(req, res, next) {
+    // TODO: users-next
     User.count(function (err, count) {
       if (err) next(err);
 
@@ -87,6 +88,7 @@ module.exports = function (app, security) {
           });
         },
         function (done) {
+          // TODO: users-next
           User.createUser(req.user, function (err, user) {
             done(err, user);
           });

From a1fc00f4667521a3d219f7a804f635c8dcf72c28 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 16 Jul 2024 11:00:33 -0600
Subject: [PATCH 010/183] [service] users next: rename authn index module to
 typescript file

---
 service/src/authentication/{index.js => index.ts} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/src/authentication/{index.js => index.ts} (100%)

diff --git a/service/src/authentication/index.js b/service/src/authentication/index.ts
similarity index 100%
rename from service/src/authentication/index.js
rename to service/src/authentication/index.ts

From e932e3739bfae7caa9c94b3cdbefe599d2a9495c Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 16 Jul 2024 11:01:28 -0600
Subject: [PATCH 011/183] [service] users next: minor formatting and todo
 comment

---
 service/src/app.api/app.api.global.ts                |  1 +
 service/src/app.api/systemInfo/app.api.systemInfo.ts | 12 +++++-------
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/service/src/app.api/app.api.global.ts b/service/src/app.api/app.api.global.ts
index 3073a901b..a0a3f1642 100644
--- a/service/src/app.api/app.api.global.ts
+++ b/service/src/app.api/app.api.global.ts
@@ -9,6 +9,7 @@ export interface AppRequestContext<Principal = unknown> {
    * variable that a Java application would use.  Node does not use the model
    * of one thread per request, so some unique value is necessary to track a
    * request context across multiple asynchronous operations.
+   * TODO: Node introduced async context tracking: https://nodejs.org/docs/latest-v18.x/api/async_context.html
    */
   readonly requestToken: unknown
   requestingPrincipal(): Principal
diff --git a/service/src/app.api/systemInfo/app.api.systemInfo.ts b/service/src/app.api/systemInfo/app.api.systemInfo.ts
index 372874fde..51c5ac1fb 100644
--- a/service/src/app.api/systemInfo/app.api.systemInfo.ts
+++ b/service/src/app.api/systemInfo/app.api.systemInfo.ts
@@ -9,21 +9,19 @@ export type ExoRedactedSystemInfo = Omit<SystemInfo, 'environment'>
 export type ExoSystemInfo = ExoPrivilegedSystemInfo | ExoRedactedSystemInfo
 
 export interface ReadSystemInfoRequest extends AppRequest {
-  context: AppRequestContext<UserWithRole>;
+  context: AppRequestContext<UserWithRole>
 }
 export interface ReadSystemInfoResponse extends AppResponse<ExoSystemInfo, InfrastructureError> {}
 
 export interface ReadSystemInfo {
-  (req: ReadSystemInfoRequest): Promise<
-    ReadSystemInfoResponse
-  >;
+  (req: ReadSystemInfoRequest): Promise<ReadSystemInfoResponse>
 }
 
 export interface SystemInfoAppLayer {
-  readSystemInfo: ReadSystemInfo;
-  permissionsService: SystemInfoPermissionService;
+  readSystemInfo: ReadSystemInfo
+  permissionsService: SystemInfoPermissionService
 }
 
 export interface SystemInfoPermissionService {
-  ensureReadSystemInfoPermission(context: AppRequestContext<UserWithRole | null>): Promise<null | PermissionDeniedError>;
+  ensureReadSystemInfoPermission(context: AppRequestContext<UserWithRole | null>): Promise<null | PermissionDeniedError>
 }
\ No newline at end of file

From 0effde548f9525822b47693aa219adaad0f2aed5 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 16 Jul 2024 13:25:53 -0600
Subject: [PATCH 012/183] [service] users next: rename authn verification
 module to typescript file

---
 service/src/authentication/{verification.js => verification.ts} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/src/authentication/{verification.js => verification.ts} (100%)

diff --git a/service/src/authentication/verification.js b/service/src/authentication/verification.ts
similarity index 100%
rename from service/src/authentication/verification.js
rename to service/src/authentication/verification.ts

From fa38816ecc2c1ff7405a391f5e9dafa528cf054a Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 17 Jul 2024 09:11:24 -0600
Subject: [PATCH 013/183] [service] users next: port verification jwt module to
 typescript

---
 service/npm-shrinkwrap.json                |  10 ++
 service/package.json                       |   1 +
 service/src/authentication/verification.ts | 179 ++++++++++-----------
 3 files changed, 94 insertions(+), 96 deletions(-)

diff --git a/service/npm-shrinkwrap.json b/service/npm-shrinkwrap.json
index a82d630e5..76571179d 100644
--- a/service/npm-shrinkwrap.json
+++ b/service/npm-shrinkwrap.json
@@ -82,6 +82,7 @@
         "@types/express-serve-static-core": "~4.17.0",
         "@types/fs-extra": "^8.0.1",
         "@types/json2csv": "~4.5.0",
+        "@types/jsonwebtoken": "^9.0.6",
         "@types/lodash": "^4.17.6",
         "@types/mocha": "^7.0.2",
         "@types/multer": "^1.4.7",
@@ -2576,6 +2577,15 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/jsonwebtoken": {
+      "version": "9.0.6",
+      "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.6.tgz",
+      "integrity": "sha512-/5hndP5dCjloafCXns6SZyESp3Ldq7YjH3zwzwczYnjxIT0Fqzk5ROSYVGfFyczIue7IUEj8hkvLbPoLQ18vQw==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/ldapjs": {
       "version": "2.2.5",
       "resolved": "https://registry.npmjs.org/@types/ldapjs/-/ldapjs-2.2.5.tgz",
diff --git a/service/package.json b/service/package.json
index b2c1794da..8f1be91e1 100644
--- a/service/package.json
+++ b/service/package.json
@@ -99,6 +99,7 @@
     "@types/express-serve-static-core": "~4.17.0",
     "@types/fs-extra": "^8.0.1",
     "@types/json2csv": "~4.5.0",
+    "@types/jsonwebtoken": "^9.0.6",
     "@types/lodash": "^4.17.6",
     "@types/mocha": "^7.0.2",
     "@types/multer": "^1.4.7",
diff --git a/service/src/authentication/verification.ts b/service/src/authentication/verification.ts
index e361929de..236d0d922 100644
--- a/service/src/authentication/verification.ts
+++ b/service/src/authentication/verification.ts
@@ -1,152 +1,139 @@
-const JWT = require('jsonwebtoken');
-
-function payload(subject, assertion) {
-  return { subject: `${subject}`, assertion: assertion };
+import JWT from 'jsonwebtoken'
+import { Json } from '../entities/entities.json_types'
+
+type Payload = {
+  subject: string | null,
+  assertion: TokenAssertion | null,
+  expiration: number | null,
+  [key: string]: Json
 }
 
-const VerificationErrorReason = {
-  NoToken: 'no_token',
-  Subject: 'subject',
-  Assertion: 'assertion',
-  Expired: 'expired',
-  Decode: 'decode',
-  Implementation: 'implementation'
+export enum VerificationErrorReason {
+  NoToken = 'no_token',
+  Subject = 'subject',
+  Assertion = 'assertion',
+  Expired = 'expired',
+  Decode = 'decode',
+  Implementation = 'implementation'
 }
 
-const TokenAssertion = {
-  Authorized: 'urn:mage:auth:authorized',
-  Captcha: 'urn:mage:signup:captcha'
+export enum TokenAssertion {
+  Authorized = 'urn:mage:auth:authorized',
+  Captcha = 'urn:mage:signup:captcha'
 }
 
-class TokenGenerateError extends Error {
+export class TokenGenerateError extends Error {
 
-  constructor(subject, assertion, secondsToLive, cause) {
-    super();
-    this.name = 'TokenVerificationError';
-    this.subject = subject;
-    this.assertion = assertion;
-    this.secondsToLive = secondsToLive;
-    this.cause = cause;
-    let causeMessage = '';
-    if (cause) {
-      causeMessage = 'cause=' + (cause instanceof Error ? `${"\n" + cause.stack || `${cause.name}: ${cause.message}`}` : `${cause}`)
-    }
-    this.message = `error generating token: subject=${subject}; assertion=${assertion}; secondsToLive=${secondsToLive}; ${causeMessage}`
+  constructor(readonly subject: string, readonly assertion: TokenAssertion, readonly secondsToLive: number, readonly cause?: Error | null) {
+    const causeMessage = cause ? `cause=\n${cause.stack || `${cause.name}: ${cause.message}`}` : ''
+    super(`error generating token: subject=${subject}; assertion=${assertion}; secondsToLive=${secondsToLive}; ${causeMessage}`)
   }
 }
 
 class TokenVerificationError extends Error {
+
   /**
    * @param reason why the verification failed
-   * @param subject the expected subject
-   * @param assertion the expected assertion
+   * @param expected the expected `Payload`
    * @param token the encoded token string
-   * @param decoded the decoded payload of the token
+   * @param decoded the decoded `Payload` of the token
    * @param cause cause of the verification error from the underlying token implementation
    */
-  constructor(reason, expected, token, decoded, cause) {
-    super();
-    this.reason = reason;
-    this.expected = expected || { subject: null, assertion: null };
-    this.token = token;
-    this.decoded = decoded;
+  constructor(readonly reason: VerificationErrorReason, readonly expected: Payload | null, readonly token: string, readonly decoded: any, readonly cause?: Error | null) {
     let reasonMessage = `${reason}`;
     let causeMessage = '';
     if (cause) {
       causeMessage = 'cause=' + (cause instanceof Error ? `${"\n" + cause.stack || `${cause.name}: ${cause.message}`}` : `${cause}`)
     }
-
-    decoded = decoded || { subject: null, assertion: null, expiration: null };
     if (reason === VerificationErrorReason.Subject) {
-      reasonMessage = `subject was ${decoded.subject}`
-    } else if (reason === VerificationErrorReason.Assertion) {
-      reasonMessage = `assertion was ${decoded.assertion}`
-    } else if (reason === VerificationErrorReason.Expired) {
-      reasonMessage = `token expired ${new Date(decoded.expiration || 0)} (${decoded.expiration})`
-    } else if (reason === VerificationErrorReason.Decode) {
-      reasonMessage = `failed to decode token`;
+      reasonMessage = `subject was ${decoded?.subject}`
     }
-
-    this.message = `TokenVerificationError (${reasonMessage}): expected subject=${this.expected.subject}, assertion=${this.expected.assertion}; token=${token}; ${causeMessage}`
+    else if (reason === VerificationErrorReason.Assertion) {
+      reasonMessage = `assertion was ${decoded?.assertion}`
+    }
+    else if (reason === VerificationErrorReason.Expired) {
+      reasonMessage = `token expired ${new Date(decoded?.expiration || 0)} (${decoded?.expiration})`
+    }
+    else if (reason === VerificationErrorReason.Decode) {
+      reasonMessage = `failed to decode token`
+    }
+    super(`TokenVerificationError (${reasonMessage}): expected subject=${expected?.subject}, assertion=${expected?.assertion}; token=${token}; ${causeMessage}`)
   }
 }
 
-const JWTService = class {
+export class JWTService {
 
-  constructor(secret, issuer) {
-    this._secret = secret;
-    this._issuer = issuer;
-    this._algorithm = 'HS256';
-  }
+  private readonly algorithm = 'HS256'
 
-  generateToken(subject, assertion, secondsToLive, claims = {}) {
-    return new Promise((resolve, reject) => {
-      const payload = Object.assign({ assertion: assertion }, claims);
+  constructor(private readonly secret: string, private readonly issuer: string) {}
 
-      JWT.sign(payload, this._secret, {
-        algorithm: this._algorithm,
-        issuer: this._issuer,
-        expiresIn: secondsToLive,
-        subject: subject,
-      },(err, token) => {
-        if (err) {
-          reject(new TokenGenerateError(subject, assertion, secondsToLive, err));
-        } else {
-          resolve(token);
+  generateToken(subject: string, assertion: TokenAssertion, secondsToLive: number, claims = {}): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const payload = Object.assign({ assertion }, claims);
+      JWT.sign(payload, this.secret,
+        {
+          algorithm: this.algorithm,
+          issuer: this.issuer,
+          expiresIn: secondsToLive,
+          subject: subject,
+        },
+        (err, token) => {
+          if (err) {
+            reject(new TokenGenerateError(subject, assertion, secondsToLive, err))
+          }
+          else {
+            resolve(String(token))
+          }
         }
-      });
-    });
+      )
+    })
   }
 
-  verifyToken(token, expected) {
+  verifyToken(token: string, expected: Payload): Promise<Payload> {
     return new Promise((resolve, reject) => {
-      JWT.verify(token, this._secret, { algorithms: [this._algorithm], issuer: this._issuer }, (err, payload) => {
+      JWT.verify(token, this.secret, { algorithms: [this.algorithm], issuer: this.issuer }, (err: JWT.VerifyErrors | null, anyPayload?: any) => {
+        const jwtPayload = anyPayload as JWT.JwtPayload
         if (!expected) {
-          expected = { subject: null, assertion: null };
+          expected = { subject: null, assertion: null, expiration: null };
         }
-
         if (err) {
           let reason = VerificationErrorReason.Implementation;
           let decoded = null;
           try {
             const jwt = JWT.decode(token, { json: true });
             if (jwt) {
-              decoded = Object.assign({}, jwt, {
+              decoded = {
+                ...jwt,
                 subject: jwt.sub || null,
                 assertion: jwt.assertion || null,
                 expiration: jwt.exp || null
-              });
+              };
             }
           } catch (err) {
             reason = VerificationErrorReason.Decode;
           }
-
           if (!decoded) {
             reason = VerificationErrorReason.Decode;
             decoded = { subject: null, assertion: null, expiration: null };
-          } else if (err instanceof JWT.TokenExpiredError) {
+          }
+          else if (err instanceof JWT.TokenExpiredError) {
             reason = VerificationErrorReason.Expired;
           }
-          return reject(new TokenVerificationError(reason, expected, token, payload, err));
-        } else if (expected.subject && payload.sub !== expected.subject) {
-          return reject(new TokenVerificationError(VerificationErrorReason.Subject, expected, token, payload))
-        } else if (expected.assertion && payload.assertion !== expected.assertion) {
-          return reject(new TokenVerificationError(VerificationErrorReason.Assertion, expected, token, payload));
+          return reject(new TokenVerificationError(reason, expected, token, jwtPayload, err));
         }
-
-        resolve(Object.assign({}, payload, {
-            subject: payload.sub || null,
-            assertion: payload.assertion || null,
-            expiration: payload.exp || null
-        }));
-
-      });
-    });
+        else if (expected.subject && jwtPayload.sub !== expected.subject) {
+          return reject(new TokenVerificationError(VerificationErrorReason.Subject, expected, token, jwtPayload))
+        }
+        else if (expected.assertion && jwtPayload.assertion !== expected.assertion) {
+          return reject(new TokenVerificationError(VerificationErrorReason.Assertion, expected, token, jwtPayload));
+        }
+        resolve({
+          ...jwtPayload,
+          subject: jwtPayload.sub || null,
+          assertion: jwtPayload.assertion || null,
+          expiration: jwtPayload.exp || null
+        })
+      })
+    })
   }
 }
-
-exports.payload = payload;
-exports.TokenAssertion = TokenAssertion;
-exports.TokenGenerateError = TokenGenerateError;
-exports.VerificationErrorReason = VerificationErrorReason;
-exports.JWTService = JWTService;

From a994cb56de0c36bcbdf418ed6e42a72c67760ea2 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 18 Jul 2024 19:41:08 -0600
Subject: [PATCH 014/183] [service] users next: rename token model to
 typescript file

---
 service/src/models/{token.js => token.ts} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/src/models/{token.js => token.ts} (100%)

diff --git a/service/src/models/token.js b/service/src/models/token.ts
similarity index 100%
rename from service/src/models/token.js
rename to service/src/models/token.ts

From 356f329fdef81c11d02423c046c981b7dc489b2e Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 22 Jul 2024 19:41:00 -0600
Subject: [PATCH 015/183] [service] users next: fixing types in token model
 module

---
 service/src/models/token.ts | 86 +++++++++++++++++++++----------------
 1 file changed, 50 insertions(+), 36 deletions(-)

diff --git a/service/src/models/token.ts b/service/src/models/token.ts
index 6c8112967..fcaeda3e0 100644
--- a/service/src/models/token.ts
+++ b/service/src/models/token.ts
@@ -1,53 +1,67 @@
-var crypto = require('crypto')
-  , mongoose = require('mongoose')
-  , environment = require('../environment/env');
+import crypto from 'crypto'
+import mongoose, { Schema } from 'mongoose'
+import { UserModel, UserDocument } from '../adapters/users/adapters.users.db.mongoose'
+import environment = require('../environment/env');
 
-// Token expiration in msecs
-var tokenExpiration = environment.tokenExpiration * 1000;
+export interface TokenDocumentAttrs {
+  token: string
+  expirationDate: Date
+  userId?: mongoose.Types.ObjectId | undefined
+  deviceId?: mongoose.Types.ObjectId | undefined
+}
+
+export type TokenDocument = mongoose.Document<mongoose.Types.ObjectId, any, TokenDocumentAttrs> & TokenDocumentAttrs
+export type TokenDocumentPopulated = TokenDocument & {
+  userId: UserDocument
+}
 
-// Creates a new Mongoose Schema object
-var Schema = mongoose.Schema;
+// Token expiration in msecs
+const tokenExpiration = environment.tokenExpiration * 1000
 
 // Collection to hold users
-var TokenSchema = new Schema({
-  userId: { type: Schema.Types.ObjectId, ref: 'User' },
-  deviceId: { type: Schema.Types.ObjectId, ref: 'Device' },
-  expirationDate: { type: Date, required: true },
-  token: { type: String, required: true }
-},{
-  versionKey: false
-});
+const TokenSchema = new Schema<TokenDocumentAttrs>(
+  {
+    userId: { type: Schema.Types.ObjectId, ref: 'User' },
+    deviceId: { type: Schema.Types.ObjectId, ref: 'Device' },
+    expirationDate: { type: Date, required: true },
+    token: { type: String, required: true }
+  },
+  { versionKey: false }
+)
 
 // TODO: index token
-TokenSchema.index({'expirationDate': 1}, {expireAfterSeconds: 0});
-
-// Creates the Model for the User Schema
-var Token = mongoose.model('Token', TokenSchema);
+TokenSchema.index({ token: 1 })
+TokenSchema.index({ expirationDate: 1 }, { expireAfterSeconds: 0 })
 
-exports.getToken = function(token, callback) {
-  Token.findOne({token: token}).populate({
-    path: 'userId',
-    populate: {
-      path: 'authenticationId',
-      model: 'Authentication'
-    }
-  }).exec(function(err, token) {
-    if (err) return callback(err);
+const Token = mongoose.model('Token', TokenSchema)
 
-    if (!token || !token.userId) {
-      return callback(null, null);
-    }
+export type SessionExpanded = {
+  token: TokenDocument,
+  user: UserDocument,
+  deviceId: mongoose.Types.ObjectId | null,
+}
 
-    token.userId.populate('roleId', function(err, user) {
-      return callback(err, {user: user, deviceId: token.deviceId, token: token});
-    });
+export async function lookupSession(token: string): Promise<SessionExpanded | null> {
+  // TODO: try to eliminate these populate join queries here or elsewhere; something is doing redundant queries
+  const tokenDoc = await Token.findOne({ token }).populate({
+    path: 'userId',
+    // populate: {
+    //   path: 'authenticationId',
+    //   model: 'Authentication'
+    // }
+  })
+  if (!tokenDoc || !tokenDoc.userId) {
+    return null
+  }
+  tokenDoc.userId.populate('roleId', function(err, user) {
+    return callback(err, {user: user, deviceId: tokenDoc.deviceId, token: tokenDoc });
   });
-};
+}
 
 exports.createToken = function(options, callback) {
   const seed = crypto.randomBytes(20);
   const token = crypto.createHash('sha256').update(seed).digest('hex');
-  const query = {userId: options.userId};
+  const query: any = { userId: options.userId };
   if (options.device) {
     query.deviceId = options.device._id;
   }

From 2b57aa70ee9c5f4c29c54d72b8750b708ec2cebc Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 22 Jul 2024 19:44:53 -0600
Subject: [PATCH 016/183] [service] users next: rename token model module to db
 adapter scheme

---
 .../adapters.authentication.db.mongoose.ts}                       | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/src/{models/token.ts => authentication/adapters.authentication.db.mongoose.ts} (100%)

diff --git a/service/src/models/token.ts b/service/src/authentication/adapters.authentication.db.mongoose.ts
similarity index 100%
rename from service/src/models/token.ts
rename to service/src/authentication/adapters.authentication.db.mongoose.ts

From e4daebe27181a239c2aaebaddfbb276b041690ab Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 23 Jul 2024 09:06:25 -0600
Subject: [PATCH 017/183] [service] users next: add passport bearer types

---
 service/npm-shrinkwrap.json | 76 +++++++++++++++++++++++++++++++++++++
 service/package.json        |  1 +
 2 files changed, 77 insertions(+)

diff --git a/service/npm-shrinkwrap.json b/service/npm-shrinkwrap.json
index 76571179d..1b8505970 100644
--- a/service/npm-shrinkwrap.json
+++ b/service/npm-shrinkwrap.json
@@ -89,6 +89,7 @@
         "@types/node": "^18.18.4",
         "@types/node-fetch": "^2.5.4",
         "@types/passport": "^1.0.3",
+        "@types/passport-http-bearer": "^1.0.41",
         "@types/sinon": "^9.0.4",
         "@types/sinon-chai": "^3.2.4",
         "@types/superagent": "^8.1.3",
@@ -2437,6 +2438,15 @@
         "url": "https://opencollective.com/turf"
       }
     },
+    "node_modules/@types/accepts": {
+      "version": "1.3.7",
+      "resolved": "https://registry.npmjs.org/@types/accepts/-/accepts-1.3.7.tgz",
+      "integrity": "sha512-Pay9fq2lM2wXPWbteBsRAGiWH2hig4ZE2asK+mm7kUzlxRTfL961rj89I6zV/E3PcIkDqyuBEcMxFT7rccugeQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/archiver": {
       "version": "5.3.4",
       "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-5.3.4.tgz",
@@ -2502,12 +2512,30 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/content-disposition": {
+      "version": "0.5.8",
+      "resolved": "https://registry.npmjs.org/@types/content-disposition/-/content-disposition-0.5.8.tgz",
+      "integrity": "sha512-QVSSvno3dE0MgO76pJhmv4Qyi/j0Yk9pBp0Y7TJ2Tlj+KCgJWY6qX7nnxCOLkZ3VYRSIk1WTxCvwUSdx6CCLdg==",
+      "dev": true
+    },
     "node_modules/@types/cookiejar": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz",
       "integrity": "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==",
       "dev": true
     },
+    "node_modules/@types/cookies": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/@types/cookies/-/cookies-0.9.0.tgz",
+      "integrity": "sha512-40Zk8qR147RABiQ7NQnBzWzDcjKzNrntB5BAmeGCb2p/MIyOE+4BVvc17wumsUqUw00bJYqoXFHYygQnEFh4/Q==",
+      "dev": true,
+      "dependencies": {
+        "@types/connect": "*",
+        "@types/express": "*",
+        "@types/keygrip": "*",
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/debug": {
       "version": "4.1.12",
       "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
@@ -2558,6 +2586,12 @@
       "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.14.tgz",
       "integrity": "sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg=="
     },
+    "node_modules/@types/http-assert": {
+      "version": "1.5.5",
+      "resolved": "https://registry.npmjs.org/@types/http-assert/-/http-assert-1.5.5.tgz",
+      "integrity": "sha512-4+tE/lwdAahgZT1g30Jkdm9PzFRde0xwxBNUyRsCitRvCQB90iuA2uJYdUnhnANRcqGXaWOGY4FEoxeElNAK2g==",
+      "dev": true
+    },
     "node_modules/@types/http-errors": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz",
@@ -2586,6 +2620,37 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/keygrip": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.6.tgz",
+      "integrity": "sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ==",
+      "dev": true
+    },
+    "node_modules/@types/koa": {
+      "version": "2.15.0",
+      "resolved": "https://registry.npmjs.org/@types/koa/-/koa-2.15.0.tgz",
+      "integrity": "sha512-7QFsywoE5URbuVnG3loe03QXuGajrnotr3gQkXcEBShORai23MePfFYdhz90FEtBBpkyIYQbVD+evKtloCgX3g==",
+      "dev": true,
+      "dependencies": {
+        "@types/accepts": "*",
+        "@types/content-disposition": "*",
+        "@types/cookies": "*",
+        "@types/http-assert": "*",
+        "@types/http-errors": "*",
+        "@types/keygrip": "*",
+        "@types/koa-compose": "*",
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/koa-compose": {
+      "version": "3.2.8",
+      "resolved": "https://registry.npmjs.org/@types/koa-compose/-/koa-compose-3.2.8.tgz",
+      "integrity": "sha512-4Olc63RY+MKvxMwVknCUDhRQX1pFQoBZ/lXcRLP69PQkEpze/0cr8LNqJQe5NFb/b19DWi2a5bTi2VAlQzhJuA==",
+      "dev": true,
+      "dependencies": {
+        "@types/koa": "*"
+      }
+    },
     "node_modules/@types/ldapjs": {
       "version": "2.2.5",
       "resolved": "https://registry.npmjs.org/@types/ldapjs/-/ldapjs-2.2.5.tgz",
@@ -2662,6 +2727,17 @@
         "@types/express": "*"
       }
     },
+    "node_modules/@types/passport-http-bearer": {
+      "version": "1.0.41",
+      "resolved": "https://registry.npmjs.org/@types/passport-http-bearer/-/passport-http-bearer-1.0.41.tgz",
+      "integrity": "sha512-ecW+9e8C+0id5iz3YZ+uIarsk/vaRPkKSajt1i1Am66t0mC9gDfQDKXZz9fnPOW2xKUufbmCSou4005VM94Feg==",
+      "dev": true,
+      "dependencies": {
+        "@types/express": "*",
+        "@types/koa": "*",
+        "@types/passport": "*"
+      }
+    },
     "node_modules/@types/passport-strategy": {
       "version": "0.2.38",
       "resolved": "https://registry.npmjs.org/@types/passport-strategy/-/passport-strategy-0.2.38.tgz",
diff --git a/service/package.json b/service/package.json
index 8f1be91e1..94eed6571 100644
--- a/service/package.json
+++ b/service/package.json
@@ -106,6 +106,7 @@
     "@types/node": "^18.18.4",
     "@types/node-fetch": "^2.5.4",
     "@types/passport": "^1.0.3",
+    "@types/passport-http-bearer": "^1.0.41",
     "@types/sinon": "^9.0.4",
     "@types/sinon-chai": "^3.2.4",
     "@types/superagent": "^8.1.3",

From 5126cf2cccde20e6a67a5b13529cd22fe0e76019 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 24 Jul 2024 08:54:13 -0600
Subject: [PATCH 018/183] [service] users next: rename token model (not
 collection) to session and add repository

---
 .../adapters.authentication.db.mongoose.ts    | 120 ++++++++----------
 1 file changed, 51 insertions(+), 69 deletions(-)

diff --git a/service/src/authentication/adapters.authentication.db.mongoose.ts b/service/src/authentication/adapters.authentication.db.mongoose.ts
index fcaeda3e0..4ab7ba73f 100644
--- a/service/src/authentication/adapters.authentication.db.mongoose.ts
+++ b/service/src/authentication/adapters.authentication.db.mongoose.ts
@@ -1,94 +1,76 @@
 import crypto from 'crypto'
 import mongoose, { Schema } from 'mongoose'
-import { UserModel, UserDocument } from '../adapters/users/adapters.users.db.mongoose'
-import environment = require('../environment/env');
+import { UserDocumentExpanded } from '../adapters/users/adapters.users.db.mongoose'
+import { UserId } from '../entities/users/entities.users'
 
-export interface TokenDocumentAttrs {
+export interface SessionDocument {
   token: string
   expirationDate: Date
   userId?: mongoose.Types.ObjectId | undefined
   deviceId?: mongoose.Types.ObjectId | undefined
 }
-
-export type TokenDocument = mongoose.Document<mongoose.Types.ObjectId, any, TokenDocumentAttrs> & TokenDocumentAttrs
-export type TokenDocumentPopulated = TokenDocument & {
-  userId: UserDocument
+export type SessionDocumentExpanded = SessionDocument & {
+  userId: UserDocumentExpanded
+  deviceId?: mongoose.Document
 }
+export type SessionModel = mongoose.Model<SessionDocument>
 
-// Token expiration in msecs
-const tokenExpiration = environment.tokenExpiration * 1000
-
-// Collection to hold users
-const TokenSchema = new Schema<TokenDocumentAttrs>(
+const SessionSchema = new Schema<SessionDocument, SessionModel>(
   {
+    token: { type: String, required: true },
     userId: { type: Schema.Types.ObjectId, ref: 'User' },
     deviceId: { type: Schema.Types.ObjectId, ref: 'Device' },
     expirationDate: { type: Date, required: true },
-    token: { type: String, required: true }
   },
   { versionKey: false }
 )
 
 // TODO: index token
-TokenSchema.index({ token: 1 })
-TokenSchema.index({ expirationDate: 1 }, { expireAfterSeconds: 0 })
+SessionSchema.index({ token: 1 })
+SessionSchema.index({ expirationDate: 1 }, { expireAfterSeconds: 0 })
 
-const Token = mongoose.model('Token', TokenSchema)
+export interface SessionRepository {
+  readonly model: SessionModel
+  createOrRefreshSession(userId: UserId, deviceId?: string): Promise<SessionDocumentExpanded>
+  removeSession(token: string): Promise<void>
+  removeSessionsForUser(userId: UserId): Promise<number>
+  removeSessionForDevice(deviceId: string): Promise<number>
+}
 
-export type SessionExpanded = {
-  token: TokenDocument,
-  user: UserDocument,
-  deviceId: mongoose.Types.ObjectId | null,
+const populateSessionUserRole: mongoose.PopulateOptions = {
+  path: 'userId',
+  populate: 'roleId'
 }
 
-export async function lookupSession(token: string): Promise<SessionExpanded | null> {
-  // TODO: try to eliminate these populate join queries here or elsewhere; something is doing redundant queries
-  const tokenDoc = await Token.findOne({ token }).populate({
-    path: 'userId',
-    // populate: {
-    //   path: 'authenticationId',
-    //   model: 'Authentication'
-    // }
+export function createSessionRepository(conn: mongoose.Connection, collectionName: string, sessionTimeoutSeconds: number): SessionRepository {
+  const model = conn.model('Token', SessionSchema, collectionName)
+  return Object.freeze({
+    model,
+    async createOrRefreshSession(userId: UserId, deviceId?: string): Promise<mongoose.HydratedDocument<SessionDocumentExpanded>> {
+      const seed = crypto.randomBytes(20)
+      const token = crypto.createHash('sha256').update(seed).digest('hex')
+      const query: any = { userId: new mongoose.Types.ObjectId(userId) }
+      if (deviceId) {
+        query.deviceId = new mongoose.Types.ObjectId(deviceId)
+      }
+      const now = Date.now()
+      const update = {
+        token,
+        expirationDate: new Date(now + sessionTimeoutSeconds * 1000)
+      }
+      return await model.findOneAndUpdate(query, update,
+        { upsert: true, new: true, populate: populateSessionUserRole })
+    },
+    async removeSession(token: string): Promise<void> {
+      await model.deleteOne({ token })
+    },
+    async removeSessionsForUser(userId: UserId): Promise<number> {
+      const { deletedCount } = await model.deleteMany({ userId: new mongoose.Types.ObjectId(userId) })
+      return deletedCount
+    },
+    async removeSessionForDevice(deviceId: string): Promise<number> {
+      const { deletedCount } = await model.deleteMany({ deviceId: new mongoose.Types.ObjectId(deviceId) })
+      return deletedCount
+    }
   })
-  if (!tokenDoc || !tokenDoc.userId) {
-    return null
-  }
-  tokenDoc.userId.populate('roleId', function(err, user) {
-    return callback(err, {user: user, deviceId: tokenDoc.deviceId, token: tokenDoc });
-  });
 }
-
-exports.createToken = function(options, callback) {
-  const seed = crypto.randomBytes(20);
-  const token = crypto.createHash('sha256').update(seed).digest('hex');
-  const query: any = { userId: options.userId };
-  if (options.device) {
-    query.deviceId = options.device._id;
-  }
-  const now = Date.now();
-  const update = {
-    token: token,
-    expirationDate: new Date(now + tokenExpiration)
-  };
-  Token.findOneAndUpdate(query, update, {upsert: true, new: true}, function(err, newToken) {
-    callback(err, newToken);
-  });
-};
-
-exports.removeToken = function(token, callback) {
-  Token.findByIdAndRemove(token._id, function(err) {
-    callback(err);
-  });
-};
-
-exports.removeTokensForUser = function(user, callback) {
-  Token.remove({userId: user._id}, function(err, numberRemoved) {
-    callback(err, numberRemoved);
-  });
-};
-
-exports.removeTokenForDevice = function(device, callback) {
-  Token.remove({deviceId: device._id}, function(err, numberRemoved) {
-    callback(err, numberRemoved);
-  });
-};

From 66b2a07c17d39a8a0a89d43448265009717f6110 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 25 Jul 2024 09:04:32 -0600
Subject: [PATCH 019/183] [service] users next: remove attrs type parameter on
 base mongoose repository and use the mongoose doctype parameter as intended
 to represent raw db documents instead of a hydrated mongoose document-model
 instance

---
 .../base/adapters.base.db.mongoose.ts         | 67 ++++++++++---------
 1 file changed, 35 insertions(+), 32 deletions(-)

diff --git a/service/src/adapters/base/adapters.base.db.mongoose.ts b/service/src/adapters/base/adapters.base.db.mongoose.ts
index 2df137d0d..306d304c5 100644
--- a/service/src/adapters/base/adapters.base.db.mongoose.ts
+++ b/service/src/adapters/base/adapters.base.db.mongoose.ts
@@ -6,72 +6,75 @@ type EntityReference = { id: string | number }
 /**
  * Map Mongoose `Document` instances to plain entity objects.
  */
-export type DocumentMapping<D extends mongoose.Document, E extends object> = (doc: D) => E
+export type DocumentMapping<DocType extends mongoose.AnyObject, E extends object> = (doc: DocType | mongoose.HydratedDocument<DocType>) => E
 /**
- * Map entities to objects suitable to create Mongoose `Document` instances, as
+ * Map entities to objects suitable to create Mongoose `Model` `Document` instances, as
  * in `new mongoose.Model(stub)`.
  */
-export type EntityMapping<D extends mongoose.Document, E extends object> = (entity: Partial<E>) => any
+export type EntityMapping<DocType extends mongoose.AnyObject, E extends object> = (entity: Partial<E>) => DocType
 
 /**
- * Return a document mapping that calls `toJSON()` on the given `Document`
+ * Return a document mapping that calls {@link mongoose.Document.toObject()} on the given `Document`
  * instance and returns the result.
  */
-export function createDefaultDocMapping<D extends mongoose.Document, E extends object>(): DocumentMapping<D, E> {
-  return (d): any => d.toJSON()
+export function createDefaultDocMapping<DocType extends mongoose.AnyObject>(): DocumentMapping<DocType, any> {
+  return d => {
+    if (d instanceof mongoose.Document) {
+      return d.toObject()
+    }
+    return d
+  }
 }
 
 /**
  * Return an entity mapping that simply returns the given entity object as is.
  */
-export function createDefaultEntityMapping<D extends mongoose.Document, E extends object>(): EntityMapping<D, E> {
+export function createDefaultEntityMapping<DocType extends mongoose.AnyObject, E extends object>(): EntityMapping<DocType, E> {
   return e => e as any
 }
 
 /**
- * * Type parameter `D` is a subtype of `mongoose.Document`
- * * Type parameter `M` is a subtpye of `mongoose.Model<D>` that creates
- *   instances of type `D`.
- * * Type parameter `Attrs` is the entity attributes type, which is typically a
- *   plain object interface, and is the type that repository queries return
- *   using `entityForDocuent()`.
- * * Type parameter `Entity` is an optional, typically more objected-oriented
- *   entity type that provides extra functionality beyond just the raw data
- *   of the `Attrs` type.
+ * * Type parameter `DocType` is the shape of the raw document the MongoDB driver stores and retrieves.
+ * * Type parameter `Model` is a subtpye of `mongoose.Model<D>` that creates instances of type `D`.
+ * * Type parameter `Entity` is the entity attributes type, which is typically a plain object interface,
+ *   or an instantiable class and is the type that repository queries return using `entityForDocument()`.
  */
-export class BaseMongooseRepository<D extends mongoose.Document, M extends mongoose.Model<D>, Attrs extends object, Entity extends object = Attrs> {
+export class BaseMongooseRepository<DocType extends mongoose.AnyObject, Model extends mongoose.Model<DocType> = mongoose.Model<DocType>, Entity extends object = DocType> {
 
-  readonly model: M
-  readonly entityForDocument: DocumentMapping<D, Attrs>
-  readonly documentStubForEntity: EntityMapping<D, Attrs>
+  readonly model: Model
+  readonly entityForDocument: DocumentMapping<DocType, Entity>
+  readonly documentStubForEntity: EntityMapping<DocType, Entity>
 
-  constructor(model: M, mapping?: { docToEntity?: DocumentMapping<D, Attrs>, entityToDocStub?: EntityMapping<D, Attrs> }) {
+  /**
+   * When the caller omits `docToEntity` and/or `entityToDocStub`
+   */
+  constructor(model: Model, mapping?: { docToEntity?: DocumentMapping<DocType, Entity>, entityToDocStub?: EntityMapping<DocType, Entity> }) {
     this.model = model
     this.entityForDocument = mapping?.docToEntity || createDefaultDocMapping()
     this.documentStubForEntity = mapping?.entityToDocStub || createDefaultEntityMapping()
   }
 
-  async create(attrs: Partial<Attrs>): Promise<Attrs> {
+  async create(attrs: Partial<Entity>): Promise<Entity> {
     const stub = this.documentStubForEntity(attrs)
     const created = await this.model.create(stub)
     return this.entityForDocument(created)
   }
 
-  async findAll(): Promise<Attrs[]> {
+  async findAll(): Promise<Entity[]> {
     const docs = await this.model.find().cursor()
-    const entities: Attrs[] = []
+    const entities: Entity[] = []
     for await (const doc of docs) {
       entities.push(this.entityForDocument(doc))
     }
     return entities
   }
 
-  async findById(id: any): Promise<Attrs | null> {
+  async findById(id: any): Promise<Entity | null> {
     const doc = await this.model.findById(id)
     return doc ? this.entityForDocument(doc) : null as any
   }
 
-  async findAllByIds<ID>(ids: ID[]): Promise<ID extends string ? { [id: string]: Attrs | null } : ID extends number ? { [id: number]: Attrs | null } : never> {
+  async findAllByIds<ID>(ids: ID[]): Promise<ID extends string ? { [id: string]: Entity | null } : ID extends number ? { [id: number]: Entity | null } : never> {
     if (!ids.length) {
       return {} as any
     }
@@ -79,16 +82,17 @@ export class BaseMongooseRepository<D extends mongoose.Document, M extends mongo
       notFound[id] = null
       return notFound
     }, {} as any)
-    const docs: D[] = await this.model.find({ _id: { $in: ids }}).exec();
+    const docs = await this.model.find({ _id: { $in: ids }})
     const found = {} as any
     for (const doc of docs) {
-      found[doc.id] = this.entityForDocument(doc)
+      const _id = typeof doc._id === 'number' || doc._id === 'string' ? doc._id : String(doc._id)
+      found[_id] = this.entityForDocument(doc)
       delete notFound[doc.id]
     }
     return { ...notFound, ...found }
   }
 
-  async update(attrs: Partial<Attrs> & EntityReference): Promise<Attrs | null> {
+  async update(attrs: Partial<Entity> & EntityReference): Promise<Entity | null> {
     let doc = (await this.model.findById(attrs.id))
     if (!doc) {
       throw new Error(`document not found for id: ${attrs.id}`)
@@ -99,7 +103,7 @@ export class BaseMongooseRepository<D extends mongoose.Document, M extends mongo
     return this.entityForDocument(doc)
   }
 
-  async removeById(id: any): Promise<Attrs | null> {
+  async removeById(id: any): Promise<Entity | null> {
     const doc = await this.model.findByIdAndRemove(id)
     if (doc) {
       return this.entityForDocument(doc)
@@ -109,8 +113,7 @@ export class BaseMongooseRepository<D extends mongoose.Document, M extends mongo
 }
 
 export const pageQuery = <RT, DT>(query: mongoose.Query<RT, DT>, paging: PagingParameters): Promise<{ totalCount: number | null, query: mongoose.Query<RT, DT> }> => {
-  //TODO had to use any to construct
-  const BaseQuery: any = query.toConstructor()
+  const BaseQuery = query.toConstructor()
   const pageQuery = new BaseQuery().limit(paging.pageSize).skip(paging.pageIndex * paging.pageSize) as mongoose.Query<RT, DT>
   const includeTotalCount = typeof paging.includeTotalCount === 'boolean' ? paging.includeTotalCount : paging.pageIndex === 0
   if (includeTotalCount) {

From ae9775bd099e27315b07b06cea41b97b70a737a4 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 26 Jul 2024 12:13:11 -0600
Subject: [PATCH 020/183] [service] users next: correct type defs in feeds and
 icons mongoose adapters

---
 .../feeds/adapters.feeds.db.mongoose.ts       | 54 +++++++++++--------
 .../icons/adapters.icons.db.mongoose.ts       | 35 ++++++------
 2 files changed, 49 insertions(+), 40 deletions(-)

diff --git a/service/src/adapters/feeds/adapters.feeds.db.mongoose.ts b/service/src/adapters/feeds/adapters.feeds.db.mongoose.ts
index ee013448e..4efbecf29 100644
--- a/service/src/adapters/feeds/adapters.feeds.db.mongoose.ts
+++ b/service/src/adapters/feeds/adapters.feeds.db.mongoose.ts
@@ -1,9 +1,8 @@
 
-import mongoose, { Model, SchemaOptions } from 'mongoose'
+import mongoose, { Model } from 'mongoose'
 import { BaseMongooseRepository } from '../base/adapters.base.db.mongoose'
 import { FeedServiceType, FeedService, FeedServiceTypeId, RegisteredFeedServiceType, FeedRepository, Feed, FeedServiceId } from '../../entities/feeds/entities.feeds'
 import { FeedServiceTypeRepository, FeedServiceRepository } from '../../entities/feeds/entities.feeds'
-import { FeedServiceDescriptor } from '../../app.api/feeds/app.api.feeds'
 import { EntityIdFactory } from '../../entities/entities.global'
 
 
@@ -13,28 +12,26 @@ export const FeedsModels = {
   Feed: 'Feed',
 }
 
-export type FeedServiceTypeIdentity = Pick<FeedServiceType, 'pluginServiceTypeId'> & {
-  id: string
+export type FeedServiceTypeIdentityDocument = Pick<FeedServiceType, 'pluginServiceTypeId'> & {
+  _id: mongoose.Types.ObjectId
   moduleName: string
 }
-export type FeedServiceTypeIdentityDocument = FeedServiceTypeIdentity & mongoose.Document
 export type FeedServiceTypeIdentityModel = Model<FeedServiceTypeIdentityDocument>
-//TODO remove cast to any
-export const FeedServiceTypeIdentitySchema = new mongoose.Schema<any>({
+export const FeedServiceTypeIdentitySchema = new mongoose.Schema<FeedServiceTypeIdentityDocument, FeedServiceTypeIdentityModel>({
   pluginServiceTypeId: { type: String, required: true },
   moduleName: { type: String, required: true }
 })
 export function FeedServiceTypeIdentityModel(conn: mongoose.Connection, collection?: string): FeedServiceTypeIdentityModel {
-  //TODO remove cast to any
-  return conn.model(FeedsModels.FeedServiceTypeIdentity, FeedServiceTypeIdentitySchema, collection || 'feed_service_types') as any
+  return conn.model(FeedsModels.FeedServiceTypeIdentity, FeedServiceTypeIdentitySchema, collection || 'feed_service_types')
 }
 
-export type FeedServiceDocument = Omit<FeedServiceDescriptor, 'serviceType'> & mongoose.Document & {
+export type FeedServiceDocument = Omit<FeedService, 'id' | 'serviceType' | 'config'> & {
+  _id: mongoose.Types.ObjectId
   serviceType: mongoose.Types.ObjectId
+  config: object
 }
 export type FeedServiceModel = Model<FeedServiceDocument>
-//TODO remove cast to any
-export const FeedServiceSchema = new mongoose.Schema<any>(
+export const FeedServiceSchema = new mongoose.Schema<FeedServiceDocument, FeedServiceModel>(
   {
     serviceType: { type: mongoose.SchemaTypes.ObjectId, required: true, ref: FeedsModels.FeedServiceTypeIdentity },
     title: { type: String, required: true },
@@ -45,7 +42,7 @@ export const FeedServiceSchema = new mongoose.Schema<any>(
     toJSON: {
       getters: true,
       versionKey: false,
-      transform: (doc: FeedServiceDocument, json: any & FeedService, options: SchemaOptions): void => {
+      transform: (doc: FeedServiceDocument, json: any & FeedService): void => {
         delete json._id
         json.serviceType = doc.serviceType.toHexString()
       }
@@ -56,13 +53,20 @@ export function FeedServiceModel(conn: mongoose.Connection, collection?: string)
   return conn.model(FeedsModels.FeedService, FeedServiceSchema, collection || 'feed_services') as any
 }
 
-export type FeedDocument = Omit<Feed, 'service' | 'icon'> & mongoose.Document & {
-  service: mongoose.Types.ObjectId,
+export type FeedDocument = Omit<Feed, 'id' | 'service' | 'icon' | 'constantParams'> & {
+  _id: string
+  service: mongoose.Types.ObjectId
   icon?: string
+  /**
+   * This extra definition with `object` avoids the following TS error on the `FeedSchema` `constantParams` member
+   * below.
+   *
+   * _Type instantiation is excessively deep and possibly infinite.ts(2589)_
+   */
+  constantParams: object
 }
 export type FeedModel = Model<FeedDocument>
-//TODO remove cast to any
-export const FeedSchema = new mongoose.Schema<any>(
+export const FeedSchema = new mongoose.Schema<FeedDocument, FeedModel>(
   {
     _id: { type: String, required: true },
     service: { type: mongoose.SchemaTypes.ObjectId, required: true, ref: FeedsModels.FeedService },
@@ -86,7 +90,7 @@ export const FeedSchema = new mongoose.Schema<any>(
     toJSON: {
       getters: true,
       versionKey: false,
-      transform: (doc: FeedDocument, json: any & Feed, options: SchemaOptions): void => {
+      transform: (doc: FeedDocument, json: any & Feed): void => {
         delete json._id
         json.service = doc.service.toHexString()
         if (doc.icon) {
@@ -96,8 +100,7 @@ export const FeedSchema = new mongoose.Schema<any>(
     }
   })
 export function FeedModel(conn: mongoose.Connection, collection?: string): FeedModel {
-  //TODO remove cast to any
-  return conn.model(FeedsModels.Feed, FeedSchema, collection || 'feeds') as any
+  return conn.model(FeedsModels.Feed, FeedSchema, collection || 'feeds')
 }
 
 export class MongooseFeedServiceTypeRepository implements FeedServiceTypeRepository {
@@ -148,7 +151,16 @@ export class MongooseFeedServiceRepository extends BaseMongooseRepository<FeedSe
 export class MongooseFeedRepository extends BaseMongooseRepository<FeedDocument, FeedModel, Feed> implements FeedRepository {
 
   constructor(model: FeedModel, private readonly idFactory: EntityIdFactory) {
-    super(model, { entityToDocStub: e => ({ ...e, icon: e.icon?.id  }) })
+    super(model, {
+      entityToDocStub: e => {
+        return {
+          ...e,
+          _id: e.id,
+          service: new mongoose.Types.ObjectId(e.service!),
+          icon: e.icon?.id
+        }
+      }
+    })
   }
 
   async create(attrs: Partial<Feed>): Promise<Feed> {
diff --git a/service/src/adapters/icons/adapters.icons.db.mongoose.ts b/service/src/adapters/icons/adapters.icons.db.mongoose.ts
index c28bd52a6..2af6daf8d 100644
--- a/service/src/adapters/icons/adapters.icons.db.mongoose.ts
+++ b/service/src/adapters/icons/adapters.icons.db.mongoose.ts
@@ -5,12 +5,13 @@ import { EntityIdFactory, pageOf, PageOf, PagingParameters, UrlResolutionError,
 import { StaticIcon, StaticIconStub, StaticIconId, StaticIconRepository, LocalStaticIconStub, StaticIconReference, StaticIconContentStore, StaticIconImportFetch } from '../../entities/icons/entities.icons'
 import { BaseMongooseRepository, pageQuery } from '../base/adapters.base.db.mongoose'
 
-export type StaticIconDocument = Omit<StaticIcon, 'sourceUrl'> & mongoose.Document & {
+export type StaticIconDocument = Omit<StaticIcon, 'sourceUrl'> & {
+  _id: string
   sourceUrl: string
 }
 export type StaticIconModel = mongoose.Model<StaticIconDocument>
 export const StaticIconModelName = 'StaticIcon'
-export const StaticIconSchema = new mongoose.Schema<StaticIconDocument>(
+export const StaticIconSchema = new mongoose.Schema<StaticIconDocument, StaticIconModel>(
   {
     _id: { type: String, required: true },
     sourceUrl: { type: String, required: true, unique: true },
@@ -38,7 +39,7 @@ export const StaticIconSchema = new mongoose.Schema<StaticIconDocument>(
     toJSON: {
       getters: true,
       versionKey: false,
-      transform: (doc: StaticIconDocument, json: any & StaticIcon, options: mongoose.SchemaOptions): void => {
+      transform: (doc: StaticIconDocument, json: any & StaticIcon): void => {
         delete json._id
         json.sourceUrl = new URL(doc.sourceUrl)
       }
@@ -85,24 +86,20 @@ export class MongooseStaticIconRepository extends BaseMongooseRepository<StaticI
       const _id = await this.idFactory.nextId()
       doc = await this.model.create({ _id, registeredTimestamp: Date.now(), ...stub })
     }
-    switch (fetch) {
-      case StaticIconImportFetch.EagerAwait:
-        const stored = await this.fetchAndStore(doc)
-        if (stored instanceof UrlResolutionError) {
-          return stored
-        }
-        doc = stored
-        break
-      case StaticIconImportFetch.Eager:
-        this.fetchAndStore(doc)
-        break
-      case StaticIconImportFetch.Lazy:
-      default:
+    if (fetch === StaticIconImportFetch.EagerAwait) {
+      const stored = await this.fetchAndStore(doc)
+      if (stored instanceof UrlResolutionError) {
+        return stored
+      }
+      doc = stored
+    }
+    else if (StaticIconImportFetch.Eager) {
+      this.fetchAndStore(doc)
     }
     return this.entityForDocument(doc)
   }
 
-  private async fetchAndStore(iconDoc: StaticIconDocument): Promise<StaticIconDocument | UrlResolutionError> {
+  private async fetchAndStore(iconDoc: mongoose.HydratedDocument<StaticIconDocument>): Promise<mongoose.HydratedDocument<StaticIconDocument> | UrlResolutionError> {
     if (typeof iconDoc.resolvedTimestamp === 'number') {
       return iconDoc
     }
@@ -193,12 +190,12 @@ export class MongooseStaticIconRepository extends BaseMongooseRepository<StaticI
     return pageOf(items, paging, counted.totalCount)
   }
 
-  private async findDocBySourceUrl(url: URL): Promise<StaticIconDocument | null> {
+  private async findDocBySourceUrl(url: URL): Promise<mongoose.HydratedDocument<StaticIconDocument> | null> {
     return await this.model.findOne({ sourceUrl: url.toString() })
   }
 }
 
-async function updateRegisteredIconIfChanged(this: MongooseStaticIconRepository, registered: StaticIconDocument, stub: StaticIconStub): Promise<StaticIconDocument> {
+async function updateRegisteredIconIfChanged(this: MongooseStaticIconRepository, registered: mongoose.HydratedDocument<StaticIconDocument>, stub: StaticIconStub): Promise<mongoose.HydratedDocument<StaticIconDocument>> {
   /*
   TODO: some of this logic could potentially be captured as an entity layer
   function, such as which properties a client is allowed to update when

From 71d1a92c61a7673d8b27cc4cd4e3cfa19686a71b Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 26 Jul 2024 12:14:03 -0600
Subject: [PATCH 021/183] [service] users next: allow partial mapped doc type
 value in base mongoose repository entity mapping

---
 service/src/adapters/base/adapters.base.db.mongoose.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/adapters/base/adapters.base.db.mongoose.ts b/service/src/adapters/base/adapters.base.db.mongoose.ts
index 306d304c5..4062ee7a5 100644
--- a/service/src/adapters/base/adapters.base.db.mongoose.ts
+++ b/service/src/adapters/base/adapters.base.db.mongoose.ts
@@ -11,7 +11,7 @@ export type DocumentMapping<DocType extends mongoose.AnyObject, E extends object
  * Map entities to objects suitable to create Mongoose `Model` `Document` instances, as
  * in `new mongoose.Model(stub)`.
  */
-export type EntityMapping<DocType extends mongoose.AnyObject, E extends object> = (entity: Partial<E>) => DocType
+export type EntityMapping<DocType extends mongoose.AnyObject, E extends object> = (entity: Partial<E>) => Partial<DocType>
 
 /**
  * Return a document mapping that calls {@link mongoose.Document.toObject()} on the given `Document`

From ecb76c106b358052579c676faaa506033f5430ad Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 26 Jul 2024 13:30:43 -0600
Subject: [PATCH 022/183] [service] users next: sync types in plugin state
 mongoose adapter to proper mongoose type patterns

---
 .../plugins/adapters.plugins.db.mongoose.ts   | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/service/src/adapters/plugins/adapters.plugins.db.mongoose.ts b/service/src/adapters/plugins/adapters.plugins.db.mongoose.ts
index d78141682..0130a4d74 100644
--- a/service/src/adapters/plugins/adapters.plugins.db.mongoose.ts
+++ b/service/src/adapters/plugins/adapters.plugins.db.mongoose.ts
@@ -2,9 +2,9 @@ import mongoose from 'mongoose'
 import { EnsureJson } from '../../entities/entities.json_types'
 import { PluginStateRepository } from '../../plugins.api'
 
-export type PluginStateDocument<State extends object> = mongoose.Document & {
+export type PluginStateDocument<State extends object> = {
   _id: 'string',
-  state: State
+  state: State | null
 }
 
 const SCHEMA_SPEC = {
@@ -14,18 +14,19 @@ const SCHEMA_SPEC = {
 
 export class MongoosePluginStateRepository<State extends object> implements PluginStateRepository<State> {
 
-  //TODO remove cast to any, was mongoose.Model<PluginStateDocument<State>>
-  readonly model: mongoose.Model<any>
+  readonly model: mongoose.Model<PluginStateDocument<State>>
 
   constructor(public readonly pluginId: string, public readonly mongoose: mongoose.Mongoose) {
     const collectionName = `plugin_state_${pluginId}`
     const modelNames = mongoose.modelNames()
-    this.model = modelNames.includes(collectionName) ? mongoose.model(collectionName) : mongoose.model(collectionName, new mongoose.Schema(SCHEMA_SPEC), collectionName)
+    this.model = modelNames.includes(collectionName) ?
+      mongoose.model<PluginStateDocument<State>>(collectionName) :
+      mongoose.model(collectionName, new mongoose.Schema<PluginStateDocument<State>>(SCHEMA_SPEC), collectionName)
   }
 
-  async put(state: EnsureJson<State>): Promise<EnsureJson<State>> {
+  async put(state: EnsureJson<State> | null): Promise<EnsureJson<State>> {
     const updated = await this.model.findByIdAndUpdate(this.pluginId, { state }, { new: true, upsert: true, setDefaultsOnInsert: false })
-    return updated.toJSON().state
+    return updated.toJSON().state as any
   }
 
   async patch(state: Partial<EnsureJson<State>>): Promise<EnsureJson<State>> {
@@ -40,11 +41,11 @@ export class MongoosePluginStateRepository<State extends object> implements Plug
       return update
     }, {} as any)
     const patched = await this.model.findByIdAndUpdate(this.pluginId, update, { new: true, upsert: true, setDefaultsOnInsert: false })
-    return patched.toJSON().state
+    return patched.toJSON().state as any
   }
 
   async get(): Promise<EnsureJson<State> | null> {
     const doc = await this.model.findById(this.pluginId)
-    return doc?.toJSON().state || null
+    return doc?.toJSON().state as any || null
   }
 }
\ No newline at end of file

From 73959e9205cc32538b79484ba7f66259abccda50 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 26 Jul 2024 13:31:24 -0600
Subject: [PATCH 023/183] [service] users next: add doc comment note about
 casting _id in mongoose queries

---
 service/src/adapters/base/adapters.base.db.mongoose.ts | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/service/src/adapters/base/adapters.base.db.mongoose.ts b/service/src/adapters/base/adapters.base.db.mongoose.ts
index 4062ee7a5..03c9938a9 100644
--- a/service/src/adapters/base/adapters.base.db.mongoose.ts
+++ b/service/src/adapters/base/adapters.base.db.mongoose.ts
@@ -74,6 +74,11 @@ export class BaseMongooseRepository<DocType extends mongoose.AnyObject, Model ex
     return doc ? this.entityForDocument(doc) : null as any
   }
 
+  /**
+   * _NOTE_ this currently only explicitly supports 'string' and 'number' IDs, though Mongoose will implicitly
+   * [cast](https://mongoosejs.com/docs/6.x/docs/tutorials/query_casting.html) a filter's `_id` entry to `ObjectID`
+   * before sending the query through the driver.
+   */
   async findAllByIds<ID>(ids: ID[]): Promise<ID extends string ? { [id: string]: Entity | null } : ID extends number ? { [id: number]: Entity | null } : never> {
     if (!ids.length) {
       return {} as any

From 1d549a919f6a7cdb94c50ba89d60021b11de809e Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 26 Jul 2024 16:48:08 -0600
Subject: [PATCH 024/183] [service] users next: sync types in mage event
 mongoose adapter to proper mongoose type patterns

---
 .../events/adapters.events.db.mongoose.ts     | 30 +++++++++++--------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/service/src/adapters/events/adapters.events.db.mongoose.ts b/service/src/adapters/events/adapters.events.db.mongoose.ts
index 4027628e1..7c910f41f 100644
--- a/service/src/adapters/events/adapters.events.db.mongoose.ts
+++ b/service/src/adapters/events/adapters.events.db.mongoose.ts
@@ -1,4 +1,4 @@
-import { BaseMongooseRepository } from '../base/adapters.base.db.mongoose'
+import { BaseMongooseRepository, DocumentMapping } from '../base/adapters.base.db.mongoose'
 import { MageEventRepository, MageEventAttrs, MageEventId, MageEvent } from '../../entities/events/entities.events'
 import mongoose from 'mongoose'
 import { FeedId } from '../../entities/feeds/entities.feeds'
@@ -10,34 +10,38 @@ export type MageEventDocument = legacy.MageEventDocument
 export type MageEventModel = mongoose.Model<legacy.MageEventDocument>
 export const MageEventSchema = legacy.Model.schema
 
-export class MongooseMageEventRepository extends BaseMongooseRepository<MageEventDocument, MageEventModel, MageEventAttrs, MageEvent> implements MageEventRepository {
+const docToEntity: DocumentMapping<MageEventDocument, MageEvent> = doc => {
+  if (doc instanceof mongoose.Document) {
+    return new MageEvent(doc.toJSON())
+  }
+  // TODO: might need to implement this
+  throw new Error('event document mapping only support hydrated model instances')
+}
+
+export class MongooseMageEventRepository extends BaseMongooseRepository<MageEventDocument, MageEventModel, MageEvent> implements MageEventRepository {
 
   constructor(model: MageEventModel) {
-    super(model)
+    super(model, { docToEntity })
   }
 
-  async create(): Promise<MageEventAttrs> {
+  async create(): Promise<MageEvent> {
     throw new Error('method not allowed')
   }
 
-  async update(attrs: Partial<MageEventAttrs> & { id: MageEventId }): Promise<MageEventAttrs | null> {
+  async update(attrs: Partial<MageEventAttrs> & { id: MageEventId }): Promise<MageEvent | null> {
     throw new Error('method not allowed')
   }
 
   async findById(id: MageEventId): Promise<MageEvent | null> {
-    const attrs = await super.findById(id)
-    if (attrs) {
-      return new MageEvent(attrs)
-    }
-    return null
+    return await super.findById(id)
   }
 
   async findActiveEvents(): Promise<MageEventAttrs[]> {
-    const docs: legacy.MageEventDocument[] = await this.model.find({ complete: { $in: [ null, false ] }}).exec()
+    const docs = await this.model.find({ complete: { $in: [ null, false ] }})
     return docs.map(this.entityForDocument)
   }
 
-  async addFeedsToEvent(event: MageEventId, ...feeds: FeedId[]): Promise<MageEventAttrs | null> {
+  async addFeedsToEvent(event: MageEventId, ...feeds: FeedId[]): Promise<MageEvent | null> {
     const updated = await this.model.findByIdAndUpdate(
       event,
       {
@@ -49,7 +53,7 @@ export class MongooseMageEventRepository extends BaseMongooseRepository<MageEven
     return updated ? this.entityForDocument(updated) : null
   }
 
-  async removeFeedsFromEvent(event: MageEventId, ...feeds: FeedId[]): Promise<MageEventAttrs | null> {
+  async removeFeedsFromEvent(event: MageEventId, ...feeds: FeedId[]): Promise<MageEvent | null> {
     const updated = await this.model.findByIdAndUpdate(
       event,
       {

From 78c6ffba26cbb42c57b0dc21eaf1357ad4b2ebec Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 29 Jul 2024 08:36:44 -0600
Subject: [PATCH 025/183] [service] users next: more mage event types

---
 .../events/adapters.events.db.mongoose.ts     |  2 +-
 .../src/entities/events/entities.events.ts    | 10 ++---
 service/src/models/event.d.ts                 | 45 +++++++++++--------
 service/src/models/event.js                   | 23 +++-------
 4 files changed, 39 insertions(+), 41 deletions(-)

diff --git a/service/src/adapters/events/adapters.events.db.mongoose.ts b/service/src/adapters/events/adapters.events.db.mongoose.ts
index 7c910f41f..728e4a164 100644
--- a/service/src/adapters/events/adapters.events.db.mongoose.ts
+++ b/service/src/adapters/events/adapters.events.db.mongoose.ts
@@ -36,7 +36,7 @@ export class MongooseMageEventRepository extends BaseMongooseRepository<MageEven
     return await super.findById(id)
   }
 
-  async findActiveEvents(): Promise<MageEventAttrs[]> {
+  async findActiveEvents(): Promise<MageEvent[]> {
     const docs = await this.model.find({ complete: { $in: [ null, false ] }})
     return docs.map(this.entityForDocument)
   }
diff --git a/service/src/entities/events/entities.events.ts b/service/src/entities/events/entities.events.ts
index c5d238815..967b1e619 100644
--- a/service/src/entities/events/entities.events.ts
+++ b/service/src/entities/events/entities.events.ts
@@ -175,21 +175,21 @@ export interface Acl {
 export type MageEventCreateAttrs = Pick<MageEventAttrs, 'name' | 'description'>
 
 export interface MageEventRepository {
-  findAll(): Promise<MageEventAttrs[]>
+  findAll(): Promise<MageEvent[]>
   findById(id: MageEventId): Promise<MageEvent | null>
-  findAllByIds(ids: MageEventId[]): Promise<{ [id: number]: MageEventAttrs | null }>
+  findAllByIds(ids: MageEventId[]): Promise<{ [id: number]: MageEvent | null }>
   /**
    * Return all the MAGE events that are not {@link MageEventAttrs.complete | complete}.
    */
-  findActiveEvents(): Promise<MageEventAttrs[]>
+  findActiveEvents(): Promise<MageEvent[]>
   /**
    * Add a reference to the given feed ID on the given event.
    * @param event an Event ID
    * @param feed a Feed ID
    */
-  addFeedsToEvent(event: MageEventId, ...feeds: FeedId[]): Promise<MageEventAttrs | null>
+  addFeedsToEvent(event: MageEventId, ...feeds: FeedId[]): Promise<MageEvent | null>
   findTeamsInEvent(event: MageEventId): Promise<Team[] | null>
-  removeFeedsFromEvent(event: MageEventId, ...feeds: FeedId[]): Promise<MageEventAttrs | null>
+  removeFeedsFromEvent(event: MageEventId, ...feeds: FeedId[]): Promise<MageEvent | null>
   /**
    * Remove the given feeds from any events that reference the feed.  Return the
    * count of events the operation modified.
diff --git a/service/src/models/event.d.ts b/service/src/models/event.d.ts
index bc1947599..664c5f0de 100644
--- a/service/src/models/event.d.ts
+++ b/service/src/models/event.d.ts
@@ -21,9 +21,8 @@ export type TeamDocument = Omit<Team, 'acl' | 'userIds'> & mongoose.Document & {
   }
 }
 
-export type MageEventDocument = Omit<mongoose.Document, '_id' | 'id'> & Omit<MageEventAttrs, 'id' | 'forms' | 'teamIds' | 'layerIds' | 'acl'> & {
+export type MageEventDocument = Omit<MageEventAttrs, 'id' | 'forms' | 'teamIds' | 'layerIds' | 'acl'> & {
   _id: number
-  id: number
   /**
    * The event's collection name is the name of the MongoDB collection that
    * stores observations for the event.
@@ -33,22 +32,32 @@ export type MageEventDocument = Omit<mongoose.Document, '_id' | 'id'> & Omit<Mag
   teamIds: mongoose.Types.ObjectId[] | TeamDocument[]
   layerIds: mongoose.Types.ObjectId[]
   acl: MageEventDocumentAcl
-  toObject(options?: MageEventDocumentToObjectOptions): any
-  toJSON(options: ToObjectOptions): MageEventAttrs
+}
+
+export type MageEventModel = mongoose.Model<MageEventDocument, object,
+  {
+    toObject(options?: MageEventDocumentToObjectOptions): MageEventAttrs
+    toJSON(options: MageEventDocumentToObjectOptions): MageEventAttrs
+  }
+>
+
+export type MageEventModelInstance = mongoose.HydratedDocument<MageEventDocument> & {
+  toObject(options?: MageEventDocumentToObjectOptions): MageEventAttrs
+  toJSON(options: MageEventDocumentToObjectOptions): MageEventAttrs
 }
 
 export interface MageEventDocumentAcl {
   [userId: string]: EventRole
 }
 
-export type FormDocument = Omit<Form, 'fields'> & mongoose.Document & {
+export type FormDocument = Omit<Form, 'fields'> & {
   _id: number
   fields: FormFieldDocument[]
 }
-export type FormFieldDocument = FormField & mongoose.Document & {
+export type FormFieldDocument = FormField & {
   _id: never
 }
-export type FormFieldChoiceDocument = FormFieldChoice & mongoose.Document & {
+export type FormFieldChoiceDocument = FormFieldChoice & {
   _id: never
 }
 
@@ -56,19 +65,19 @@ export type TODO = any
 export type Callback<Result = unknown> = (err: Error | null, result?: Result) => void
 
 export declare function count(options: TODO, callback: Callback<number>): void
-export declare function getEvents(options: TODO, callback: Callback<MageEventDocument[]>): void
-export declare function getById(id: MageEventId, options: TODO, callback: Callback<MageEventDocument | null>): void
-export declare function filterEventsByUserId(events: MageEventDocument[], userId: string, callback: Callback<MageEventDocument[]>): void
-export declare function create(event: MageEventCreateAttrs, user: Partial<UserDocument> & Pick<UserDocument, '_id'>, callback: Callback<MageEventDocument>): void
-export declare function addForm(eventId: MageEventId, form: any, callback: Callback<MageEventDocument>): void
-export declare function addLayer(event: MageEventDocument, layer: any, callback: Callback<MageEventDocument>): void
-export declare function removeLayer(event: MageEventDocument, layer: { id: any }, callback: Callback<MageEventDocument>): void
+export declare function getEvents(options: TODO, callback: Callback<MageEventModelInstance[]>): void
+export declare function getById(id: MageEventId, options: TODO, callback: Callback<MageEventModelInstance | null>): void
+export declare function filterEventsByUserId(events: MageEventDocument[], userId: string, callback: Callback<MageEventModelInstance[]>): void
+export declare function create(event: MageEventCreateAttrs, user: Partial<UserDocument> & Pick<UserDocument, '_id'>, callback: Callback<MageEventModelInstance>): void
+export declare function addForm(eventId: MageEventId, form: any, callback: Callback<MageEventModelInstance>): void
+export declare function addLayer(event: MageEventDocument, layer: any, callback: Callback<MageEventModelInstance>): void
+export declare function removeLayer(event: MageEventDocument, layer: { id: any }, callback: Callback<MageEventModelInstance>): void
 export declare function getUsers(eventId: MageEventId, callback: Callback<UserDocument[]>): void
-export declare function addTeam(event: MageEventDocument, team: any, callback: Callback<MageEventDocument>): void
+export declare function addTeam(event: MageEventDocument, team: any, callback: Callback<MageEventModelInstance>): void
 export declare function getTeams(eventId: MageEventId, options: { populate: string[] | null }, callback: Callback): void
-export declare function removeTeam(event: MageEventDocument, team: any, callback: Callback<MageEventDocument>): void
-export declare function updateUserInAcl(eventId: MageEventId, userId: string, role: string, callback: Callback<MageEventDocument>): void
-export declare function removeUserFromAcl(eventId: MageEventId, userId: string, callback: Callback<MageEventDocument>): void
+export declare function removeTeam(event: MageEventDocument, team: any, callback: Callback<MageEventModelInstance>): void
+export declare function updateUserInAcl(eventId: MageEventId, userId: string, role: string, callback: Callback<MageEventModelInstance>): void
+export declare function removeUserFromAcl(eventId: MageEventId, userId: string, callback: Callback<MageEventModelInstance>): void
 export declare function getMembers(eventId: MageEventId, options: TODO): Promise<PageInfo>
 export declare function getNonMembers(eventId: MageEventId, options: TODO): Promise<PageInfo>
 export declare function getTeamsInEvent(eventId: MageEventId, options: TODO): Promise<PageInfo>
diff --git a/service/src/models/event.js b/service/src/models/event.js
index 1f0e8c804..44cf590ee 100644
--- a/service/src/models/event.js
+++ b/service/src/models/event.js
@@ -10,7 +10,6 @@ const mongoose = require('mongoose')
   , log = require('winston')
 const { rolesWithPermission, EventRolePermissions } = require('../entities/events/entities.events');
 
-// Creates a new Mongoose Schema object
 const Schema = mongoose.Schema;
 
 const OptionSchema = new Schema({
@@ -158,8 +157,7 @@ EventSchema.pre('init', function (event) {
   instance during the pre-init hook's execution.
   */
   if (event.forms) {
-    populateUserFields(event, function () {
-    });
+    populateUserFields(event, function () {});
   }
 });
 
@@ -208,6 +206,7 @@ function transform(event, ret, options) {
   }
 
   // if read only permissions in event acl, only return users acl
+  // TODO: move this business logic
   if (options.access) {
     const roleOfUserOnEvent = ret.acl[options.access.user._id];
     const rolesThatCanModify = rolesWithPermission('update').concat(rolesWithPermission('delete'));
@@ -234,21 +233,11 @@ function transform(event, ret, options) {
   }
 }
 
-FormSchema.set("toJSON", {
-  transform: transformForm
-});
-
-FormSchema.set("toObject", {
-  transform: transformForm
-});
+FormSchema.set("toJSON", { transform: transformForm });
+FormSchema.set("toObject", { transform: transformForm });
 
-EventSchema.set("toJSON", {
-  transform: transform
-});
-
-EventSchema.set("toObject", {
-  transform: transform
-});
+EventSchema.set("toJSON", { transform });
+EventSchema.set("toObject", { transform });
 
 // Creates the Model for the Layer Schema
 const Event = mongoose.model('Event', EventSchema);

From 6cdc1300ce43892bafb574abe9b6e8179b5226f1 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 29 Jul 2024 08:38:06 -0600
Subject: [PATCH 026/183] [service] users next: fix lint errors

---
 .../observations/adapters.observations.db.mongoose.ts         | 4 ++--
 service/src/entities/feeds/entities.feeds.ts                  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/service/src/adapters/observations/adapters.observations.db.mongoose.ts b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
index 276611156..a0e4a57d0 100644
--- a/service/src/adapters/observations/adapters.observations.db.mongoose.ts
+++ b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
@@ -118,11 +118,11 @@ export class MongooseObservationRepository extends BaseMongooseRepository<legacy
   }
 
   async nextFormEntryIds(count: number = 1): Promise<FormEntryId[]> {
-    return Array.from({ length: count }).map(_ => (new mongoose.Types.ObjectId()).toHexString())
+    return Array.from({ length: count }).map(() => (new mongoose.Types.ObjectId()).toHexString())
   }
 
   async nextAttachmentIds(count: number = 1): Promise<AttachmentId[]> {
-    return Array.from({ length: count }).map(_ => (new mongoose.Types.ObjectId()).toHexString())
+    return Array.from({ length: count }).map(() => (new mongoose.Types.ObjectId()).toHexString())
   }
 }
 
diff --git a/service/src/entities/feeds/entities.feeds.ts b/service/src/entities/feeds/entities.feeds.ts
index 412b215b6..6ec6ce72e 100644
--- a/service/src/entities/feeds/entities.feeds.ts
+++ b/service/src/entities/feeds/entities.feeds.ts
@@ -422,7 +422,7 @@ export function validateSchemaPropertyReferencesForFeed<T extends Feed | FeedCre
     return feed
   }
   const referencedProps = [ feed.itemPrimaryProperty, feed.itemSecondaryProperty, feed.itemTemporalProperty ]
-  if (referencedProps.every(key => key ? schemaProps.hasOwnProperty(key) : true)) {
+  if (referencedProps.every(key => key ? Object.prototype.hasOwnProperty.call(schemaProps, key) : true)) {
     return feed
   }
   return new FeedsError(ErrInvalidFeedAttrs, { invalidKeys: [ 'itemPropertiesSchema' ] },

From 71f57082e032523acf22125cfb96868b71cfc936 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 29 Jul 2024 08:40:16 -0600
Subject: [PATCH 027/183] [service] users next: mongoose types in base mongoose
 adapter test

---
 service/test/adapters/base/adapters.base.db.mongoose.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/test/adapters/base/adapters.base.db.mongoose.test.ts b/service/test/adapters/base/adapters.base.db.mongoose.test.ts
index b6594a712..4a80ec707 100644
--- a/service/test/adapters/base/adapters.base.db.mongoose.test.ts
+++ b/service/test/adapters/base/adapters.base.db.mongoose.test.ts
@@ -17,7 +17,7 @@ describe('mongoose adapter layer base', function() {
     squee?: boolean
     noo?: number
   }
-  type BaseDocument = BaseEntity & mongoose.Document
+  type BaseDocument = BaseEntity
   type BaseModel = mongoose.Model<BaseDocument>
 
   const collection = 'base'

From 301ee0c75cd42a83a693f4a60b50bdb4fe32183a Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 29 Jul 2024 12:00:53 -0600
Subject: [PATCH 028/183] [service] users next: remove unnecessary any cast in
 feeds mongoose adapter

---
 service/src/adapters/feeds/adapters.feeds.db.mongoose.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/service/src/adapters/feeds/adapters.feeds.db.mongoose.ts b/service/src/adapters/feeds/adapters.feeds.db.mongoose.ts
index 4efbecf29..803044ea8 100644
--- a/service/src/adapters/feeds/adapters.feeds.db.mongoose.ts
+++ b/service/src/adapters/feeds/adapters.feeds.db.mongoose.ts
@@ -49,8 +49,7 @@ export const FeedServiceSchema = new mongoose.Schema<FeedServiceDocument, FeedSe
     }
   })
 export function FeedServiceModel(conn: mongoose.Connection, collection?: string): FeedServiceModel {
-  //TODO remove cast to any
-  return conn.model(FeedsModels.FeedService, FeedServiceSchema, collection || 'feed_services') as any
+  return conn.model(FeedsModels.FeedService, FeedServiceSchema, collection || 'feed_services')
 }
 
 export type FeedDocument = Omit<Feed, 'id' | 'service' | 'icon' | 'constantParams'> & {

From b4c19f2d5988ea07c13f973b26964b42dbbf3b28 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 2 Aug 2024 16:53:23 -0600
Subject: [PATCH 029/183] [service] mongoose types: fixing observation repo

---
 .../base/adapters.base.db.mongoose.ts         |   4 +
 .../adapters.observations.db.mongoose.ts      | 283 ++++++++++++++++--
 .../observations/entities.observations.ts     |  12 +
 3 files changed, 272 insertions(+), 27 deletions(-)

diff --git a/service/src/adapters/base/adapters.base.db.mongoose.ts b/service/src/adapters/base/adapters.base.db.mongoose.ts
index 03c9938a9..c4a3a4424 100644
--- a/service/src/adapters/base/adapters.base.db.mongoose.ts
+++ b/service/src/adapters/base/adapters.base.db.mongoose.ts
@@ -3,6 +3,10 @@ import { PagingParameters } from '../../entities/entities.global'
 
 type EntityReference = { id: string | number }
 
+export type MongooseDefaultVersionKey = '__v'
+export const MongooseDefaultVersionKey: MongooseDefaultVersionKey = '__v'
+export type WithMongooseDefaultVersionKey = { [MongooseDefaultVersionKey]: number }
+
 /**
  * Map Mongoose `Document` instances to plain entity objects.
  */
diff --git a/service/src/adapters/observations/adapters.observations.db.mongoose.ts b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
index a0e4a57d0..fbbd2a31e 100644
--- a/service/src/adapters/observations/adapters.observations.db.mongoose.ts
+++ b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
@@ -1,6 +1,6 @@
-import { MageEvent, MageEventId } from '../../entities/events/entities.events'
-import { Attachment, AttachmentId, AttachmentNotFoundError, AttachmentPatchAttrs, copyObservationAttrs, EventScopedObservationRepository, FormEntry, FormEntryId, Observation, ObservationAttrs, ObservationId, ObservationImportantFlag, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationRepositoryForEvent, ObservationState, patchAttachment, Thumbnail } from '../../entities/observations/entities.observations'
-import { BaseMongooseRepository, DocumentMapping, pageQuery } from '../base/adapters.base.db.mongoose'
+import { MageEvent, MageEventAttrs, MageEventId } from '../../entities/events/entities.events'
+import { Attachment, AttachmentId, AttachmentNotFoundError, AttachmentPatchAttrs, copyObservationAttrs, EventScopedObservationRepository, FormEntry, FormEntryId, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationRepositoryForEvent, ObservationState, patchAttachment, Thumbnail } from '../../entities/observations/entities.observations'
+import { BaseMongooseRepository, DocumentMapping, pageQuery, WithMongooseDefaultVersionKey } from '../base/adapters.base.db.mongoose'
 import mongoose from 'mongoose'
 import * as legacy from '../../models/observation'
 import { MageEventDocument } from '../../models/event'
@@ -8,19 +8,248 @@ import { pageOf, PageOf, PagingParameters } from '../../entities/entities.global
 import { MongooseMageEventRepository } from '../events/adapters.events.db.mongoose'
 import { EventEmitter } from 'events'
 
-export type ObservationIdDocument = mongoose.Document
+const Schema = mongoose.Schema
+
+export type ObservationIdDocument = { _id: mongoose.Types.ObjectId }
 export type ObservationIdModel = mongoose.Model<ObservationIdDocument>
 
-export class MongooseObservationRepository extends BaseMongooseRepository<legacy.ObservationDocument, legacy.ObservationModel, ObservationAttrs> implements EventScopedObservationRepository {
+export type ObservationDocument = Omit<ObservationAttrs, 'id' | 'eventId' | 'userId' | 'deviceId' | 'important' | 'favoriteUserIds' | 'attachments' | 'states' | 'properties'>
+  & {
+    _id: mongoose.Types.ObjectId
+    userId?: mongoose.Types.ObjectId
+    deviceId?: mongoose.Types.ObjectId
+    important?: ObservationDocumentImportantFlag
+    favoriteUserIds: mongoose.Types.ObjectId[]
+    states: ObservationStateDocument[]
+    attachments: AttachmentDocument[]
+    properties: ObservationDocumentFeatureProperties
+  }
+  & WithMongooseDefaultVersionKey
+export type ObservationSubdocuments = {
+  states: mongoose.Types.DocumentArray<ObservationStateDocument>
+  attachments: mongoose.Types.DocumentArray<AttachmentDocument>
+}
+export type ObservationModelOverrides = ObservationSubdocuments & {
+  toObject(options?: ObservationTransformOptions): ObservationAttrs
+  toJSON(options?: ObservationTransformOptions): ObservationDocumentJson
+}
+export type ObservationSchema = mongoose.Schema<ObservationDocument>
+export type ObservationModel = mongoose.Model<ObservationDocument, object, ObservationModelOverrides>
+export type ObservationModelInstance = mongoose.HydratedDocument<ObservationDocument, ObservationModelOverrides>
+export type ObservationDocumentJson = Omit<ObservationAttrs, 'id' | 'eventId' | 'attachments' | 'states'> & {
+  id: mongoose.Types.ObjectId
+  eventId?: number
+  /**
+   * @deprecated TODO: confine URLs to the web layer
+   */
+  url: string
+  attachments: AttachmentDocumentJson[]
+  states: ObservationStateDocumentJson[]
+}
+
+/**
+ * This interface defines the options that one can supply to the `toJSON()`
+ * method of the Mongoose Document instances of the Observation model.
+ */
+export interface ObservationTransformOptions extends mongoose.ToObjectOptions {
+  /**
+   * The database schema does not include the event ID for observation
+   * documents.  Use this option to add the `eventId` property to the
+   * observation JSON document.
+   */
+  event?: MageEventDocument | MageEventAttrs | { id: MageEventId, [others: string]: any }
+  /**
+   * If the `path` option is present, the JSON transormation will prefix the
+   * `url` property of the observation JSON object with the value of `path`.
+   * @deprecated TODO: confine URLs to the web layer
+   */
+  path?: string
+}
+export type ObservationDocumentFeatureProperties = Omit<ObservationFeatureProperties, 'forms'> & {
+  forms: ObservationDocumentFormEntry[]
+}
+
+export type ObservationDocumentFormEntry = Omit<FormEntry, 'id'> & {
+  _id: mongoose.Types.ObjectId
+}
+export type ObservationDocumnetFormEntryJson = Omit<FormEntry, 'id'> & {
+  id: ObservationDocumentFormEntry['_id']
+}
+
+export type AttachmentDocument = Omit<Attachment, 'id' | 'observationFormId' | 'contentLocator' | 'thumbnails'> & {
+  _id: mongoose.Types.ObjectId
+  observationFormId: mongoose.Types.ObjectId
+  relativePath?: string
+  thumbnails: ThumbnailDocument[]
+}
+export type AttachmentDocumentJson = Omit<Attachment, 'id' | 'contentLocator' | 'thumbnails'> & {
+  id: AttachmentDocument['_id']
+  relativePath?: string
+  /**
+   * @deprecated TODO: confine URLs to the web layer
+   */
+  url?: string
+}
+
+export type ThumbnailDocument = Omit<Thumbnail, 'id' | 'contentLocator'> & {
+  _id: mongoose.Types.ObjectId
+  relativePath?: string
+}
 
-  readonly eventScope: MageEventId
-  readonly idModel: ObservationIdModel
+export type ObservationDocumentImportantFlag = Omit<ObservationImportantFlag, 'userId'> & {
+  userId?: mongoose.Types.ObjectId
+}
 
-  constructor(eventDoc: Pick<MageEventDocument, 'id' | 'collectionName'>, readonly eventLookup: (eventId: MageEventId) => Promise<MageEvent | null>, readonly domainEvents: EventEmitter) {
-    // TODO: do not bind to the default mongoose instance and connection
-    super(legacy.observationModel(eventDoc), { docToEntity: createDocumentMapping(eventDoc.id) })
-    this.eventScope = eventDoc.id
-    this.idModel = legacy.ObservationId
+export type ObservationStateDocument = Omit<ObservationState, 'id' | 'userId'> & {
+  _id: mongoose.Types.ObjectId
+  userId?: mongoose.Types.ObjectId
+}
+export type ObservationStateDocumentJson = Omit<ObservationState, 'id'> & {
+  id: string
+  // TODO: url should move to web layer
+  url: string
+}
+
+export const ObservationIdSchema = new Schema<ObservationIdDocument, ObservationIdModel>()
+
+function hasOwnProperty(wut: any, property: PropertyKey): boolean {
+  return Object.prototype.hasOwnProperty.call(wut, property)
+}
+
+function transformObservationModelInstance(modelInstance: mongoose.HydratedDocument<ObservationDocument, ObservationSubdocuments>, result: any, options: ObservationTransformOptions): ObservationAttrs {
+  result.id = modelInstance._id.toHexString()
+  delete result._id
+  delete result.__v
+  const event = options.event || {} as any
+  if (hasOwnProperty(event, '_id')) {
+    result.eventId = event?._id
+  }
+  else if (hasOwnProperty(event, 'id')) {
+    result.eventId = event.id
+  }
+  const path = options.path || ''
+  result.url = `${path}/${result.id}`
+  if (result.states && result.states.length) {
+    const currentState = result.states[0]
+    const currentStateId = currentState._id.toHexString()
+    result.state = {
+      id: currentStateId,
+      name: currentState.name,
+      userId: currentState.userId?.toHexString(),
+      url: `${result.url}/states/${currentStateId}`
+    }
+    delete result.states
+  }
+  if (result.properties && result.properties.forms) {
+    result.properties.forms = result.properties.forms.map((formEntry: AttachmentDocument) => {
+      const mapped: any = formEntry
+      mapped.id = formEntry._id.toHexString()
+      delete mapped._id
+      return mapped
+    })
+  }
+  if (result.attachments) {
+    result.attachments = modelInstance.attachments.map(doc => {
+      const entity = attachmentAttrsForDoc(doc) as Partial<Attachment>
+      delete entity.thumbnails
+      entity.url = `${result.url}/attachments/${entity.id}`
+      return entity
+    })
+  }
+  const populatedUserId = modelInstance.populated('userId')
+  if (populatedUserId) {
+    result.user = result.userId
+    // TODO Update mobile clients to handle observation.userId or observation.user.id
+    // Leave userId as mobile clients depend on it for observation create/update,
+    result.userId = populatedUserId
+  }
+  const populatedImportantUserId = modelInstance.populated('important.userId')
+  if (populatedImportantUserId && result.important) {
+    result.important.user = result.important.userId
+    delete result.important.userId
+  }
+  return result
+}
+
+ObservationIdSchema.set("toJSON", { transform: transformObservationModelInstance })
+
+export const StateSchema = new Schema({
+  name: { type: String, required: true },
+  userId: { type: Schema.Types.ObjectId, ref: 'User' }
+});
+
+export const ThumbnailSchema = new Schema(
+  {
+    minDimension: { type: Number, required: true },
+    contentType: { type: String, required: false },
+    size: { type: Number, required: false },
+    name: { type: String, required: false },
+    width: { type: Number, required: false },
+    height: { type: Number, required: false },
+    relativePath: { type: String, required: false },
+  },
+  { strict: false }
+)
+
+export const AttachmentSchema = new Schema(
+  {
+    observationFormId: { type: Schema.Types.ObjectId, required: true },
+    fieldName: { type: String, required: true },
+    lastModified: { type: Date, required: false },
+    contentType: { type: String, required: false },
+    size: { type: Number, required: false },
+    name: { type: String, required: false },
+    relativePath: { type: String, required: false },
+    width: { type: Number, required: false },
+    height: { type: Number, required: false },
+    oriented: { type: Boolean, required: true, default: false },
+    thumbnails: [ThumbnailSchema]
+  },
+  { strict: false }
+)
+
+export const ObservationSchema = new Schema(
+  {
+    type: { type: String, enum: ['Feature'], required: true },
+    lastModified: { type: Date, required: false },
+    userId: { type: Schema.Types.ObjectId, ref: 'User', required: false, sparse: true },
+    deviceId: { type: Schema.Types.ObjectId, required: false, sparse: true },
+    geometry: Schema.Types.Mixed,
+    properties: Schema.Types.Mixed,
+    attachments: [AttachmentSchema],
+    states: [StateSchema],
+    important: {
+      userId: { type: Schema.Types.ObjectId, ref: 'User', required: false },
+      timestamp: { type: Date, required: false },
+      description: { type: String, required: false }
+    },
+    favoriteUserIds: [{ type: Schema.Types.ObjectId, ref: 'User' }]
+  },
+  {
+    strict: false,
+    timestamps: {
+      createdAt: 'createdAt',
+      updatedAt: 'lastModified'
+    }
+  }
+)
+
+ObservationSchema.index({ geometry: '2dsphere' })
+ObservationSchema.index({ lastModified: 1 })
+ObservationSchema.index({ userId: 1 })
+ObservationSchema.index({ deviceId: 1 })
+ObservationSchema.index({ 'properties.timestamp': 1 })
+ObservationSchema.index({ 'states.name': 1 })
+ObservationSchema.index({ 'attachments.lastModified': 1 })
+ObservationSchema.index({ 'attachments.oriented': 1 })
+ObservationSchema.index({ 'attachments.contentType': 1 })
+ObservationSchema.index({ 'attachments.thumbnails.minDimension': 1 })
+
+export class MongooseObservationRepository extends BaseMongooseRepository<ObservationDocument, ObservationModel, ObservationAttrs> implements EventScopedObservationRepository {
+
+  constructor(model: ObservationModel, readonly idModel: ObservationIdModel, readonly eventScope: MageEventId, readonly eventLookup: (eventId: MageEventId) => Promise<MageEvent | null>, readonly domainEvents: EventEmitter) {
+    super(model, { docToEntity: createDocumentMapping(eventScope) })
+    this.idModel = mongoose.model('ObservationId')
   }
 
   async allocateObservationId(): Promise<ObservationId> {
@@ -64,7 +293,7 @@ export class MongooseObservationRepository extends BaseMongooseRepository<legacy
       }
       beforeDoc = new this.model(docSeed)
     }
-    const savedDoc = await beforeDoc!.save() as legacy.ObservationDocument
+    const savedDoc = await beforeDoc!.save() as ObservationDocument
     const savedAttrs = this.entityForDocument(savedDoc)
     const saved = Observation.evaluate(savedAttrs, observation.mageEvent)
     for (const e of observation.pendingEvents) {
@@ -143,14 +372,14 @@ export const createObservationRepositoryFactory = (eventRepo: MongooseMageEventR
   }
 }
 
-export function docToEntity(doc: legacy.ObservationDocument, eventId: MageEventId): ObservationAttrs {
+export function docToEntity(doc: ObservationDocument, eventId: MageEventId): ObservationAttrs {
   return createDocumentMapping(eventId)(doc)
 }
 
-function createDocumentMapping(eventId: MageEventId): DocumentMapping<legacy.ObservationDocument, ObservationAttrs> {
+function createDocumentMapping(eventId: MageEventId): DocumentMapping<ObservationDocument, ObservationAttrs> {
   return doc => {
     const attrs: ObservationAttrs = {
-      id: doc.id,
+      id: doc._id.toHexString(),
       eventId,
       createdAt: doc.createdAt,
       lastModified: doc.lastModified,
@@ -172,9 +401,9 @@ function createDocumentMapping(eventId: MageEventId): DocumentMapping<legacy.Obs
   }
 }
 
-function importantFlagAttrsForDoc(doc: legacy.ObservationDocument): ObservationImportantFlag | undefined {
+function importantFlagAttrsForDoc(doc: ObservationDocument): ObservationImportantFlag | undefined {
   /*
-  because the observation schema defines `important` as a nested documnet
+  because the observation schema defines `important` as a nested document
   instead of a subdocument schema, a mongoose observation document instance
   always returns a value for `observation.important`, even if the `important`
   key is undefined in the database.  so, if `important` is undefined in the
@@ -189,12 +418,12 @@ function importantFlagAttrsForDoc(doc: legacy.ObservationDocument): ObservationI
       description: docImportant.description
     }
   }
-  return void (0)
+  return void(0)
 }
 
-function attachmentAttrsForDoc(doc: legacy.AttachmentDocument): Attachment {
+function attachmentAttrsForDoc(doc: AttachmentDocument): Attachment {
   return {
-    id: doc.id,
+    id: doc._id.toHexString(),
     observationFormId: doc.observationFormId.toHexString(),
     fieldName: doc.fieldName,
     lastModified: doc.lastModified ? new Date(doc.lastModified) : undefined,
@@ -209,7 +438,7 @@ function attachmentAttrsForDoc(doc: legacy.AttachmentDocument): Attachment {
   }
 }
 
-function thumbnailAttrsForDoc(doc: legacy.ThumbnailDocument): Thumbnail {
+function thumbnailAttrsForDoc(doc: ThumbnailDocument): Thumbnail {
   return {
     // TODO: is id necessary for thumnails? needs cleanup
     contentLocator: doc.relativePath,
@@ -222,7 +451,7 @@ function thumbnailAttrsForDoc(doc: legacy.ThumbnailDocument): Thumbnail {
   }
 }
 
-function stateAttrsForDoc(doc: legacy.ObservationStateDocument): ObservationState {
+function stateAttrsForDoc(doc: ObservationStateDocument): ObservationState {
   return {
     id: doc.id,
     name: doc.name,
@@ -230,7 +459,7 @@ function stateAttrsForDoc(doc: legacy.ObservationStateDocument): ObservationStat
   }
 }
 
-function formEntryForDoc(doc: legacy.ObservationDocumentFormEntry): FormEntry {
+function formEntryForDoc(doc: ObservationDocumentFormEntry): FormEntry {
   const { _id, ...withoutDbId } = doc
   return {
     ...withoutDbId,
@@ -247,15 +476,15 @@ function assignMongoIdToDocAttrs(attrs: { id?: any, [other: string]: any }): { _
   return withoutId
 }
 
-function attachmentDocSeedForEntity(attrs: Attachment): legacy.AttachmentDocAttrs {
+function attachmentDocSeedForEntity(attrs: Attachment): AttachmentDocument {
   const seed = assignMongoIdToDocAttrs(attrs)
   seed.relativePath = attrs.contentLocator
   delete seed['contentLocator']
   seed.thumbnails = attrs.thumbnails.map(thumbnailDocSeedForEntity)
-  return seed as legacy.AttachmentDocAttrs
+  return seed as AttachmentDocument
 }
 
-function thumbnailDocSeedForEntity(attrs: Thumbnail): legacy.ThumbnailDocAttrs {
+function thumbnailDocSeedForEntity(attrs: Thumbnail): ThumbnailDocument {
   return {
     _id: new mongoose.Types.ObjectId(),
     relativePath: attrs.contentLocator,
diff --git a/service/src/entities/observations/entities.observations.ts b/service/src/entities/observations/entities.observations.ts
index 72955f2c3..505a89a99 100644
--- a/service/src/entities/observations/entities.observations.ts
+++ b/service/src/entities/observations/entities.observations.ts
@@ -31,6 +31,10 @@ export interface ObservationAttrs extends Feature<Geometry, ObservationFeaturePr
    *   a unique id
    */
   states: readonly ObservationState[]
+  /**
+   * @deprecated TODO: confine URLs to the web layer
+   */
+  url?: string
 }
 
 export interface ObservationFeatureProperties {
@@ -69,6 +73,10 @@ export interface ObservationState {
   id: string | PendingEntityId
   name: 'active' | 'archived'
   userId?: UserId | undefined
+  /**
+   * @deprecated TODO: confine URLs to the web layer
+   */
+  url?: string
 }
 
 export type FormEntryId = string
@@ -145,6 +153,10 @@ export interface Attachment {
    */
   oriented: boolean
   thumbnails: Thumbnail[]
+  /**
+   * @deprecated TODO: confine URLs to the web layer
+   */
+  url?: string
 }
 
 export interface Thumbnail {

From 5d041f1f8da64c3b3811f74432d127d50f94141c Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 2 Aug 2024 18:08:47 -0600
Subject: [PATCH 030/183] [service] mongoose types: fixing observation repo

---
 .../adapters.observations.db.mongoose.ts      | 38 ++++++-------------
 1 file changed, 11 insertions(+), 27 deletions(-)

diff --git a/service/src/adapters/observations/adapters.observations.db.mongoose.ts b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
index fbbd2a31e..e710ad584 100644
--- a/service/src/adapters/observations/adapters.observations.db.mongoose.ts
+++ b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
@@ -1,17 +1,20 @@
 import { MageEvent, MageEventAttrs, MageEventId } from '../../entities/events/entities.events'
-import { Attachment, AttachmentId, AttachmentNotFoundError, AttachmentPatchAttrs, copyObservationAttrs, EventScopedObservationRepository, FormEntry, FormEntryId, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationRepositoryForEvent, ObservationState, patchAttachment, Thumbnail } from '../../entities/observations/entities.observations'
+import { Attachment, AttachmentId, AttachmentNotFoundError, AttachmentPatchAttrs, copyObservationAttrs, EventScopedObservationRepository, FormEntry, FormEntryId, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationState, patchAttachment, Thumbnail } from '../../entities/observations/entities.observations'
 import { BaseMongooseRepository, DocumentMapping, pageQuery, WithMongooseDefaultVersionKey } from '../base/adapters.base.db.mongoose'
 import mongoose from 'mongoose'
-import * as legacy from '../../models/observation'
 import { MageEventDocument } from '../../models/event'
 import { pageOf, PageOf, PagingParameters } from '../../entities/entities.global'
-import { MongooseMageEventRepository } from '../events/adapters.events.db.mongoose'
 import { EventEmitter } from 'events'
 
 const Schema = mongoose.Schema
 
 export type ObservationIdDocument = { _id: mongoose.Types.ObjectId }
 export type ObservationIdModel = mongoose.Model<ObservationIdDocument>
+export const ObservationIdModelName: string = 'ObservationId'
+export const ObservationIdSchema = new Schema<ObservationIdDocument, ObservationIdModel>()
+export function createObservationIdModel(conn: mongoose.Connection): ObservationIdModel {
+  return conn.model(ObservationIdModelName, ObservationIdSchema)
+}
 
 export type ObservationDocument = Omit<ObservationAttrs, 'id' | 'eventId' | 'userId' | 'deviceId' | 'important' | 'favoriteUserIds' | 'attachments' | 'states' | 'properties'>
   & {
@@ -110,13 +113,11 @@ export type ObservationStateDocumentJson = Omit<ObservationState, 'id'> & {
   url: string
 }
 
-export const ObservationIdSchema = new Schema<ObservationIdDocument, ObservationIdModel>()
-
 function hasOwnProperty(wut: any, property: PropertyKey): boolean {
   return Object.prototype.hasOwnProperty.call(wut, property)
 }
 
-function transformObservationModelInstance(modelInstance: mongoose.HydratedDocument<ObservationDocument, ObservationSubdocuments>, result: any, options: ObservationTransformOptions): ObservationAttrs {
+function transformObservationModelInstance(modelInstance: mongoose.HydratedDocument<ObservationDocument, ObservationSubdocuments> | mongoose.HydratedDocument<ObservationIdDocument>, result: any, options: ObservationTransformOptions): ObservationAttrs {
   result.id = modelInstance._id.toHexString()
   delete result._id
   delete result.__v
@@ -149,7 +150,7 @@ function transformObservationModelInstance(modelInstance: mongoose.HydratedDocum
     })
   }
   if (result.attachments) {
-    result.attachments = modelInstance.attachments.map(doc => {
+    result.attachments = result.attachments.map((doc: AttachmentDocument) => {
       const entity = attachmentAttrsForDoc(doc) as Partial<Attachment>
       delete entity.thumbnails
       entity.url = `${result.url}/attachments/${entity.id}`
@@ -355,23 +356,6 @@ export class MongooseObservationRepository extends BaseMongooseRepository<Observ
   }
 }
 
-export const createObservationRepositoryFactory = (eventRepo: MongooseMageEventRepository, domainEvents: EventEmitter): ObservationRepositoryForEvent => {
-  return async (eventId: MageEventId): Promise<EventScopedObservationRepository> => {
-    const event = await eventRepo.model.findById(eventId)
-    if (event) {
-      return new MongooseObservationRepository(
-        { id: eventId, collectionName: event.collectionName },
-        async mageEventId => {
-          return await eventRepo.findById(mageEventId)
-        },
-        domainEvents)
-    }
-    const err = new Error(`unexpected error: event not found for id ${event}`)
-    console.error(err)
-    throw err
-  }
-}
-
 export function docToEntity(doc: ObservationDocument, eventId: MageEventId): ObservationAttrs {
   return createDocumentMapping(eventId)(doc)
 }
@@ -453,7 +437,7 @@ function thumbnailAttrsForDoc(doc: ThumbnailDocument): Thumbnail {
 
 function stateAttrsForDoc(doc: ObservationStateDocument): ObservationState {
   return {
-    id: doc.id,
+    id: doc._id.toHexString(),
     name: doc.name,
     userId: doc.userId?.toHexString()
   }
@@ -462,8 +446,8 @@ function stateAttrsForDoc(doc: ObservationStateDocument): ObservationState {
 function formEntryForDoc(doc: ObservationDocumentFormEntry): FormEntry {
   const { _id, ...withoutDbId } = doc
   return {
-    ...withoutDbId,
-    id: _id.toHexString()
+    ...withoutDbId as any,
+    id: _id.toHexString(),
   }
 }
 

From e5d554337a7de8c18542734cd420d154c418f70f Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 2 Aug 2024 18:26:47 -0600
Subject: [PATCH 031/183] [service] users next: add user to domain terms

---
 docs/domain.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/docs/domain.md b/docs/domain.md
index 8b647c36c..13b3e0a3b 100644
--- a/docs/domain.md
+++ b/docs/domain.md
@@ -44,6 +44,9 @@ Many MAGE customers have enterprise data sources available to them that are rele
 
 ## Core Terms
 
+### User
+A user is a human that interacts with the Mage application to support or accomplish a business goal.
+
 ### Feature
 A feature represents a physical object or occurrence that has spatial and/or temporal attributes.  A spatial attribute is the geographic location and shape of a feature and includes single points, (e.g. latitude and longitude), and geographic geometry structures such as lines and polygon shapes.  A temporal attribute could be a single instantaneous timestamp (e.g., 1 January 2020 at 10:35:40.555 AM) or a temporal duration (e.g., 2020-01-01 through 2020-01-31).  See **feature** from https://www.ogc.org/ogc/glossary/f.
 
@@ -97,13 +100,13 @@ An event is a scope to manage users, the data they collect, and the data they ar
 An event defines the observation data participants can submit.  Events may define one or more forms into which participants enter observation data about a subject.  Each form defines one or more form fields of varying types into which a participant enters a data value of the field's type, such as a date, text, number, email, etc.  An event may impose validation rules on submitted observations, such as minimum and/or maximum number of entries for a given form.  Form fields may impose validation rules on individual data values, such as required vs. optional, minimum and/or maximum numeric values, text input patterns, or allowed attachment media types.
 
 ### Participant
-A participant is a user that has access to the data associated with a specific event, as well as to submit observations for the event.
+A participant is a user that has access to the data associated with a specific event, as well as access to submit observations for the event.
 
 ### Field Participant
 A field participant is a participant of an event that is actively collecting observations for the event using a mobile device.
 
 ### Monitor
-A monitor is a participant of an event that is not actively collecting data for the event in the field.
+A monitor is a user that has access to view data associated with an event, but not to create or modify data for the event.
 
 ### Location
 A location is the reported geospatial position of a field participant.  Locations, therefore, only exist within the scope of an event.

From 3529f8235f9499a471d197c04c5d032f9dafe1d5 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 3 Aug 2024 09:06:30 -0600
Subject: [PATCH 032/183] refactor(service/observations): remove unused legacy
 model and api methods; TODO: remove remaining references to
 observationModel() factory function

---
 service/src/api/observation.js                |  22 +--
 .../018-set-default-password-policy.js        |   4 +-
 service/src/models/observation.d.ts           |  88 ---------
 service/src/models/observation.js             | 183 +-----------------
 4 files changed, 8 insertions(+), 289 deletions(-)

diff --git a/service/src/api/observation.js b/service/src/api/observation.js
index bc212611b..d804f1bc3 100644
--- a/service/src/api/observation.js
+++ b/service/src/api/observation.js
@@ -44,6 +44,11 @@ Observation.prototype.getAll = function(options, callback) {
   }
 };
 
+/**
+ * TODO: this can be deleted when these refs go away
+ * * routes/index.js
+ * * routes/observations.js
+ */
 Observation.prototype.getById = function(observationId, options, callback) {
   if (typeof options === 'function') {
     callback = options;
@@ -177,23 +182,6 @@ Observation.prototype.validate = function(observation) {
   }
 };
 
-Observation.prototype.createObservationId = function(callback) {
-  ObservationModel.createObservationId(callback);
-};
-
-Observation.prototype.validateObservationId = function(id, callback) {
-  ObservationModel.getObservationId(id, function(err, id) {
-    if (err) return callback(err);
-
-    if (!id) {
-      err = new Error();
-      err.status = 404;
-    }
-
-    callback(err, id);
-  });
-};
-
 // TODO create is gone, do I need to figure out if this is an observation create?
 Observation.prototype.update = function(observationId, observation, callback) {
   if (this._user) observation.userId = this._user._id;
diff --git a/service/src/migrations/018-set-default-password-policy.js b/service/src/migrations/018-set-default-password-policy.js
index 27e63d1da..62b4f0e05 100644
--- a/service/src/migrations/018-set-default-password-policy.js
+++ b/service/src/migrations/018-set-default-password-policy.js
@@ -1,7 +1,7 @@
 "use strict";
 
-const config = require('../config.js'),
-  log = require('winston');
+const config = require('../config.js');
+const log = require('winston');
 
 exports.id = 'set-default-password-policy';
 
diff --git a/service/src/models/observation.d.ts b/service/src/models/observation.d.ts
index 0297c2ef7..2cfc9e262 100644
--- a/service/src/models/observation.d.ts
+++ b/service/src/models/observation.d.ts
@@ -4,94 +4,6 @@ import { MageEventAttrs, MageEventId } from '../entities/events/entities.events'
 import { Attachment, FormEntry, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationState, Thumbnail } from '../entities/observations/entities.observations'
 import { MageEventDocument } from './event'
 
-export type ObservationDocument = Omit<mongoose.Document, 'toJSON'> & Omit<ObservationAttrs, 'eventId' | 'userId' | 'deviceId' | 'important' | 'favoriteUserIds' | 'attachments' | 'states' | 'properties'> & {
-  userId?: mongoose.Types.ObjectId
-  deviceId?: mongoose.Types.ObjectId
-  important?: ObservationDocumentImportantFlag
-  favoriteUserIds: mongoose.Types.ObjectId[]
-  states: ObservationStateDocument[]
-  attachments: AttachmentDocument[]
-  properties: ObservationDocumentProperties
-  toJSON(options?: ObservationJsonOptions): ObservationDocumentJson
-}
-export interface ObservationModel extends mongoose.Model<ObservationDocument> {}
-export type ObservationDocumentJson = Omit<ObservationAttrs, 'id' | 'eventId' | 'attachments' | 'states'> & {
-  id: mongoose.Types.ObjectId
-  eventId?: number
-  url: string
-  attachments: AttachmentDocumentJson[]
-  states: ObservationStateDocumentJson[]
-}
-
-/**
- * This interface defines the options that one can supply to the `toJSON()`
- * method of the Mongoose Document instances of the Observation model.
- */
-export interface ObservationJsonOptions extends mongoose.ToObjectOptions {
-  /**
-   * The database schema does not include the event ID for observation
-   * documents.  Use this option to add the `eventId` property to the
-   * observation JSON document.
-   */
-  event?: MageEventDocument | MageEventAttrs | { id: MageEventId, [others: string]: any }
-  /**
-   * If the `path` option is prenent, the JSON transormation will prefix the
-   * `url` property of the observation JSON object with the value of `path`.
-   */
-  path?: string
-}
-
-export type ObservationDocumentProperties = Omit<ObservationFeatureProperties, 'forms'> & {
-  forms: ObservationDocumentFormEntry[]
-}
-
-export type ObservationDocumentFormEntry = FormEntry & {
-  _id: mongoose.Types.ObjectId
-}
-export type ObservationDocumnetFormEntryJson = Omit<FormEntry, 'id'> & {
-  id: ObservationDocumentFormEntry['_id']
-}
-
-export type AttachmentDocAttrs = Omit<Attachment, 'id' | 'observationFormId' | 'contentLocator' | 'thumbnails'> & {
-  _id: mongoose.Types.ObjectId
-  observationFormId: mongoose.Types.ObjectId
-  relativePath?: string
-  thumbnails: ThumbnailDocAttrs[]
-}
-export type AttachmentDocument = mongoose.Document & Omit<AttachmentDocAttrs, 'thumbnails'> & {
-  thumbnails: ThumbnailDocument[]
-}
-export type AttachmentDocumentJson = Omit<Attachment, 'id' | 'contentLocator' | 'thumbnails'> & {
-  id: AttachmentDocument['_id']
-  relativePath?: string
-  url?: string
-}
-
-export type ThumbnailDocAttrs = Omit<Thumbnail, 'id' | 'contentLocator'> & {
-  _id: mongoose.Types.ObjectId
-  relativePath?: string
-}
-export type ThumbnailDocument = mongoose.Document & ThumbnailDocAttrs
-
-export type ObservationDocumentImportantFlag = Omit<ObservationImportantFlag, 'userId'> & {
-  userId?: mongoose.Types.ObjectId
-}
-
-export type ObservationStateDocument = Omit<mongoose.Document & ObservationState, 'id' | 'userId'> & {
-  id: ObservationState['id']
-  userId?: mongoose.Types.ObjectId
-}
-export type ObservationStateDocumentJson = Omit<ObservationState, 'id'> & {
-  id: ObservationStateDocument['_id']
-  url: string
-}
-
-export const ObservationIdSchema: mongoose.Schema
-export type ObservationIdDocument = mongoose.Document
-export const ObservationId: mongoose.Model<ObservationIdDocument>
-
-export function observationModel(event: Partial<MageEventDocument> & Pick<MageEventDocument, 'collectionName'>): ObservationModel
-
 export interface ObservationReadOptions {
   filter?: {
     geometry?: Geometry
diff --git a/service/src/models/observation.js b/service/src/models/observation.js
index 45c53839a..9a9440eee 100644
--- a/service/src/models/observation.js
+++ b/service/src/models/observation.js
@@ -3,160 +3,8 @@ const mongoose = require('mongoose')
   , Event = require('./event')
   , log = require('winston');
 
-const Schema = mongoose.Schema;
-
-// Collection to hold unique observation ids
-const ObservationIdSchema = new Schema();
-ObservationIdSchema.set("toJSON", { transform });
-const ObservationId = mongoose.model('ObservationId', ObservationIdSchema);
-
-exports.ObservationIdSchema = ObservationIdSchema;
-exports.ObservationId = ObservationId;
-
-const StateSchema = new Schema({
-  name: { type: String, required: true },
-  userId: { type: Schema.Types.ObjectId, ref: 'User' }
-});
-
-const ThumbnailSchema = new Schema({
-  minDimension: { type: Number, required: true },
-  contentType: { type: String, required: false },
-  size: { type: Number, required: false },
-  name: { type: String, required: false },
-  width: { type: Number, required: false },
-  height: { type: Number, required: false },
-  relativePath: { type: String, required: false },
-}, {
-  strict: false
-});
-
-const AttachmentSchema = new Schema({
-  observationFormId: { type: Schema.Types.ObjectId, required: true },
-  fieldName: { type: String, required: true },
-  lastModified: { type: Date, required: false },
-  contentType: { type: String, required: false },
-  size: { type: Number, required: false },
-  name: { type: String, required: false },
-  relativePath: { type: String, required: false },
-  width: { type: Number, required: false },
-  height: { type: Number, required: false },
-  oriented: { type: Boolean, required: true, default: false },
-  thumbnails: [ThumbnailSchema]
-}, {
-  strict: false
-});
-
-// Creates the Schema for the Attachments object
-const ObservationSchema = new Schema({
-  type: { type: String, enum: ['Feature'], required: true },
-  lastModified: { type: Date, required: false },
-  userId: { type: Schema.Types.ObjectId, ref: 'User', required: false, sparse: true },
-  deviceId: { type: Schema.Types.ObjectId, required: false, sparse: true },
-  geometry: Schema.Types.Mixed,
-  properties: Schema.Types.Mixed,
-  attachments: [AttachmentSchema],
-  states: [StateSchema],
-  important: {
-    userId: { type: Schema.Types.ObjectId, ref: 'User', required: false },
-    timestamp: { type: Date, required: false },
-    description: { type: String, required: false }
-  },
-  favoriteUserIds: [{ type: Schema.Types.ObjectId, ref: 'User' }]
-}, {
-  strict: false,
-  timestamps: {
-    createdAt: 'createdAt',
-    updatedAt: 'lastModified'
-  }
-});
-
-ObservationSchema.index({ geometry: '2dsphere' });
-ObservationSchema.index({ lastModified: 1 });
-ObservationSchema.index({ userId: 1 });
-ObservationSchema.index({ deviceId: 1 });
-ObservationSchema.index({ 'properties.timestamp': 1 });
-ObservationSchema.index({ 'states.name': 1 });
-ObservationSchema.index({ 'attachments.lastModified': 1 });
-ObservationSchema.index({ 'attachments.oriented': 1 });
-ObservationSchema.index({ 'attachments.contentType': 1 });
-ObservationSchema.index({ 'attachments.thumbnails.minDimension': 1 });
-
-function transformAttachment(attachment, observation) {
-  attachment.id = attachment._id;
-  delete attachment._id;
-  delete attachment.thumbnails;
-
-  /*
-  TODO: is this actually checking if the attachment is stored?
-  ANSWER: yes it is. the clients depend on it. the web client will not upload
-    if the url property is present on an attachment.
-    stop sending absolute urls to the clients.
-  */
-  if (attachment.relativePath) {
-    attachment.url = [observation.url, "attachments", attachment.id].join("/");
-  }
-
-  return attachment;
-}
-
-function transformState(state, observation) {
-  state.id = state._id;
-  delete state._id;
-
-  state.url = [observation.url, "states", state.id].join("/");
-  return state;
-}
-
-function transform(observation, ret, options) {
-  ret.id = ret._id;
-  delete ret._id;
-  delete ret.__v;
-
-  if (options.event) {
-    ret.eventId = options.event._id;
-  }
-
-  const path = options.path ? options.path : "";
-  ret.url = [path, observation.id].join("/");
-
-  if (observation.states && observation.states.length) {
-    ret.state = transformState(ret.states[0], ret);
-    delete ret.states;
-  }
-
-  if (observation.properties && observation.properties.forms) {
-    ret.properties.forms = ret.properties.forms.map(observationForm => {
-      observationForm.id = observationForm._id;
-      delete observationForm._id;
-      return observationForm;
-    });
-  }
-
-  if (observation.attachments) {
-    ret.attachments = ret.attachments.map(function (attachment) {
-      return transformAttachment(attachment, ret);
-    });
-  }
 
-  const populatedUserId = observation.populated('userId');
-  if (populatedUserId) {
-    ret.user = ret.userId;
-    // TODO Update mobile clients to handle observation.userId or observation.user.id
-    // Leave userId as mobile clients depend on it for observation create/update,
-    ret.userId = populatedUserId;
-  }
-
-  const populatedImportantUserId = observation.populated('important.userId')
-  if (populatedImportantUserId && ret.important) {
-    ret.important.user = ret.important.userId
-    delete ret.important.userId;
-  }
-}
-
-ObservationSchema.set('toJSON', { transform });
-ObservationSchema.set('toObject', { transform });
-
-const models = {};
+// TODO: are these necessary?
 mongoose.model('Attachment', AttachmentSchema);
 mongoose.model('Thumbnail', ThumbnailSchema);
 mongoose.model('State', StateSchema);
@@ -201,21 +49,6 @@ function parseFields(fields) {
   }
 }
 
-function observationModel(event) {
-  const name = event.collectionName;
-  let model = models[name];
-  if (!model) {
-    // Creates the Model for the Observation Schema
-    model = mongoose.model(name, ObservationSchema, name);
-    // TODO: mongoose should be caching these models so this seems unnecessary
-    models[name] = model;
-  }
-
-  return model;
-}
-
-exports.observationModel = observationModel;
-
 exports.getObservations = function (event, o, callback) {
   const conditions = {};
 
@@ -298,20 +131,6 @@ exports.getObservations = function (event, o, callback) {
   }
 };
 
-exports.createObservationId = function (callback) {
-  ObservationId.create({}, callback);
-};
-
-exports.getObservationId = function (id, callback) {
-  ObservationId.findById(id, function (err, doc) {
-    callback(err, doc ? doc._id : null);
-  });
-};
-
-exports.getLatestObservation = function (event, callback) {
-  observationModel(event).findOne({}, { lastModified: true }, { sort: { "lastModified": -1 }, limit: 1 }, callback);
-};
-
 exports.getObservationById = function (event, observationId, options, callback) {
   const fields = parseFields(options.fields);
 

From f473088ee5731418eccba0e1d52889d896f4647b Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 3 Aug 2024 09:07:17 -0600
Subject: [PATCH 033/183] lint(service): fix type lint warnings in main module

---
 service/src/app.ts | 40 ++++++++++++++++++++++++++--------------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/service/src/app.ts b/service/src/app.ts
index f5af7a8c3..9d6413ad3 100644
--- a/service/src/app.ts
+++ b/service/src/app.ts
@@ -20,8 +20,6 @@ import { PreFetchedUserRoleFeedsPermissionService } from './permissions/permissi
 import { FeedsRoutes } from './adapters/feeds/adapters.feeds.controllers.web'
 import { WebAppRequestFactory } from './adapters/adapters.controllers.web'
 import { AppRequest, AppRequestContext } from './app.api/app.api.global'
-// TODO: users-next
-import { UserDocument } from './models/user'
 import SimpleIdFactory from './adapters/adapters.simple_id_factory'
 import { JsonSchemaService, JsonValidator, JSONSchema4 } from './entities/entities.json_types'
 import { MageEventModel, MongooseMageEventRepository } from './adapters/events/adapters.events.db.mongoose'
@@ -50,11 +48,11 @@ import { RoleBasedUsersPermissionService } from './permissions/permissions.users
 import { MongoosePluginStateRepository } from './adapters/plugins/adapters.plugins.db.mongoose'
 import path from 'path'
 import { MageEventDocument } from './models/event'
-import { parseAcceptLanguageHeader } from './entities/entities.i18n'
+import { Locale, parseAcceptLanguageHeader } from './entities/entities.i18n'
 import { ObservationRoutes, ObservationWebAppRequestFactory } from './adapters/observations/adapters.observations.controllers.web'
 import { UserWithRole } from './permissions/permissions.role-based.base'
 import { AttachmentStore, EventScopedObservationRepository, ObservationRepositoryForEvent } from './entities/observations/entities.observations'
-import { createObservationRepositoryFactory } from './adapters/observations/adapters.observations.db.mongoose'
+import { createObservationIdModel, MongooseObservationRepository, ObservationIdModel, ObservationSchema } from './adapters/observations/adapters.observations.db.mongoose'
 import { FileSystemAttachmentStoreInitError, intializeAttachmentStore } from './adapters/observations/adapters.observations.attachment_store.file_system'
 import { AttachmentStoreToken, ObservationRepositoryToken } from './plugins.api/plugins.api.observations'
 import { GetDbConnection, MongooseDbConnectionToken } from './plugins.api/plugins.api.db'
@@ -228,6 +226,9 @@ type DatabaseLayer = {
   events: {
     event: MageEventModel
   }
+  observations: {
+    observationId: ObservationIdModel
+  }
   icons: {
     staticIcon: StaticIconModel
   },
@@ -312,6 +313,9 @@ async function initDatabase(): Promise<DatabaseLayer> {
     events: {
       event: require('./models/event').Model
     },
+    observations: {
+      observationId: createObservationIdModel(conn)
+    },
     icons: {
       staticIcon: StaticIconModel(conn)
     },
@@ -388,7 +392,15 @@ async function initRepositories(models: DatabaseLayer, config: BootConfig): Prom
       eventRepo
     },
     observations: {
-      obsRepoFactory: createObservationRepositoryFactory(eventRepo, DomainEvents),
+      obsRepoFactory: async (eventId): Promise<EventScopedObservationRepository> => {
+        const eventModelInstance = await models.events.event.findById(eventId)
+        if (!eventModelInstance) {
+          throw new Error(`unexpected error: event not found for id ${eventId}`)
+        }
+        const modelName = eventModelInstance.collectionName
+        const obsModel = models.conn.models[eventModelInstance.collectionName] || models.conn.model(modelName, ObservationSchema)
+        return new MongooseObservationRepository(obsModel, models.observations.observationId, eventId, eventRepo.findById.bind(eventRepo), DomainEvents)
+      },
       attachmentStore
     },
     icons: {
@@ -527,7 +539,7 @@ async function initSettingsAppLayer(repos: Repositories): Promise<AppLayer['sett
   }
 }
 
-interface MageEventRequestContext extends AppRequestContext<UserDocument> {
+interface MageEventRequestContext extends AppRequestContext<UserExpanded> {
   event: MageEventDocument | MageEvent | undefined
 }
 
@@ -539,7 +551,7 @@ async function initWebLayer(repos: Repositories, app: AppLayer, webUIPlugins: st
   const webLayer = await import('./express')
   const webController = webLayer.app
   const webAuth = webLayer.auth
-  const appRequestFactory: WebAppRequestFactory = <Params>(req: express.Request, params: Params): AppRequest<UserDocument, MageEventRequestContext> & Params => {
+  const appRequestFactory: WebAppRequestFactory = <Params>(req: express.Request, params: Params): AppRequest<UserExpanded, MageEventRequestContext> & Params => {
     return {
       ...params,
       context: {
@@ -623,15 +635,15 @@ async function initWebLayer(repos: Repositories, app: AppLayer, webUIPlugins: st
   const pluginAppRequestContext: GetAppRequestContext = (req: express.Request) => {
     return {
       requestToken: Symbol(),
-      requestingPrincipal() {
+      requestingPrincipal(): UserExpanded {
         /*
-        TODO: this should ideally change so that the existing passport login
+        TODO: users-next: this should ideally change so that the existing passport login
         middleware applies the entity form of a user on the request rather than
         the mongoose document instance
         */
         return { ...req.user.toJSON(), id: req.user._id.toHexString() } as UserExpanded
       },
-      locale() {
+      locale(): Locale | null {
         return Object.freeze({
           languagePreferences: parseAcceptLanguageHeader(req.headers['accept-language'])
         })
@@ -648,7 +660,7 @@ async function initWebLayer(repos: Repositories, app: AppLayer, webUIPlugins: st
   }
   return {
     webController,
-    addAuthenticatedPluginRoutes: (pluginId: string, initPluginRoutes: WebRoutesHooks['webRoutes']) => {
+    addAuthenticatedPluginRoutes: (pluginId: string, initPluginRoutes: WebRoutesHooks['webRoutes']): void => {
       const routes = initPluginRoutes(pluginAppRequestContext)
       webController.use(`/plugins/${pluginId}`, [ bearerAuth, routes ])
     }
@@ -658,10 +670,10 @@ async function initWebLayer(repos: Repositories, app: AppLayer, webUIPlugins: st
 function baseAppRequestContext(req: express.Request): AppRequestContext<UserWithRole> {
   return {
     requestToken: Symbol(),
-    requestingPrincipal() {
+    requestingPrincipal(): UserWithRole {
       return req.user as UserWithRole
     },
-    locale() {
+    locale(): Locale | null {
       return Object.freeze({
         languagePreferences: parseAcceptLanguageHeader(req.headers['accept-language'])
       })
@@ -670,7 +682,7 @@ function baseAppRequestContext(req: express.Request): AppRequestContext<UserWith
 }
 
 function ensureObservationEventScope(eventRepo: MageEventRepository, createObsRepo: ObservationRepositoryForEvent) {
-  return async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+  return async (req: express.Request, res: express.Response, next: express.NextFunction): Promise<any> => {
     const eventIdFromPath = req.params[observationEventScopeKey]
     const eventId: MageEventId = parseInt(eventIdFromPath)
     const mageEvent = Number.isInteger(eventId) ? await eventRepo.findById(eventId) : null

From 95924512f6e244cd59834fddbe408e89e91b3253 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sun, 4 Aug 2024 10:04:32 -0600
Subject: [PATCH 034/183] chore(service): transition observation read by id
 operation to DI architecture

---
 .../observations/app.api.observations.ts      |  8 +-
 .../observations/app.impl.observations.ts     | 15 +++
 service/src/routes/observations.js            | 97 ++-----------------
 ...pters.observations.controllers.web.test.ts |  7 ++
 .../app/observations/app.observations.test.ts | 32 +++++-
 5 files changed, 66 insertions(+), 93 deletions(-)

diff --git a/service/src/app.api/observations/app.api.observations.ts b/service/src/app.api/observations/app.api.observations.ts
index 1207b9054..b398304b9 100644
--- a/service/src/app.api/observations/app.api.observations.ts
+++ b/service/src/app.api/observations/app.api.observations.ts
@@ -2,7 +2,6 @@ import { EntityNotFoundError, InfrastructureError, InvalidInputError, Permission
 import { AppRequest, AppRequestContext, AppResponse } from '../app.api.global'
 import { Attachment, AttachmentId, copyObservationAttrs, EventScopedObservationRepository, FormEntry, FormFieldEntry, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationState, StagedAttachmentContentRef, Thumbnail, thumbnailIndexForTargetDimension } from '../../entities/observations/entities.observations'
 import { MageEvent } from '../../entities/events/entities.events'
-import _ from 'lodash'
 import { User, UserId } from '../../entities/users/entities.users'
 
 
@@ -33,6 +32,13 @@ export interface SaveObservationRequest extends ObservationRequest {
   observation: ExoObservationMod
 }
 
+export interface ReadObservationRequest extends ObservationRequest {
+  observationId: ObservationId
+}
+export interface ReadObservation {
+  (req: ReadObservationRequest): Promise<AppResponse<ExoObservation, PermissionDeniedError | EntityNotFoundError>>
+}
+
 export interface StoreAttachmentContent {
   (req: StoreAttachmentContentRequest): Promise<AppResponse<ExoObservation, PermissionDeniedError | EntityNotFoundError | InvalidInputError | InfrastructureError>>
 }
diff --git a/service/src/app.impl/observations/app.impl.observations.ts b/service/src/app.impl/observations/app.impl.observations.ts
index 158f398f0..6135f3dfb 100644
--- a/service/src/app.impl/observations/app.impl.observations.ts
+++ b/service/src/app.impl/observations/app.impl.observations.ts
@@ -52,6 +52,21 @@ export function SaveObservation(permissionService: api.ObservationPermissionServ
   }
 }
 
+export function ReadObservation(permissionService: api.ObservationPermissionService): api.ReadObservation {
+  return async function readObservation(req: api.ReadObservationRequest): ReturnType<api.ReadObservation> {
+    const denied = await permissionService.ensureReadObservationPermission(req.context)
+    if (denied) {
+      return AppResponse.error(denied)
+    }
+    const obs = await req.context.observationRepository.findById(req.observationId)
+    if (obs instanceof Observation) {
+      const exoObs = api.exoObservationFor(obs)
+      return AppResponse.success(exoObs)
+    }
+    return AppResponse.error(entityNotFound(req.observationId, 'Observation'))
+  }
+}
+
 export function StoreAttachmentContent(permissionService: api.ObservationPermissionService, attachmentStore: AttachmentStore): api.StoreAttachmentContent {
   return async function storeAttachmentContent(req: api.StoreAttachmentContentRequest): ReturnType<api.StoreAttachmentContent> {
     const obsRepo = req.context.observationRepository
diff --git a/service/src/routes/observations.js b/service/src/routes/observations.js
index d27ed60e3..a1d839b12 100644
--- a/service/src/routes/observations.js
+++ b/service/src/routes/observations.js
@@ -1,21 +1,15 @@
-const { attachmentTypeIsValidForField } = require('../entities/events/entities.events.forms');
-
 module.exports = function (app, security) {
 
-  const async = require('async')
-    , api = require('../api')
+  const api = require('../api')
     , log = require('winston')
     , archiver = require('archiver')
     , path = require('path')
     , environment = require('../environment/env')
-    , fs = require('fs-extra')
     , moment = require('moment')
-    , Team = require('../models/team')
     , access = require('../access')
     , { default: turfCentroid } = require('@turf/centroid')
     , geometryFormat = require('../format/geoJsonFormat')
     , observationXform = require('../transformers/observation')
-    , FileType = require('file-type')
     , passport = security.authentication.passport
     , { defaultEventPermissionsService: eventPermissions } = require('../permissions/permissions.events');
 
@@ -42,74 +36,17 @@ module.exports = function (app, security) {
     res.sendStatus(403);
   }
 
-  function validateObservationCreateAccess(validateObservationId) {
-    return function (req, res, next) {
-
+  function validateObservationCreateAccess() {
+    return async (req, res, next) => {
       if (!access.userHasPermission(req.user, 'CREATE_OBSERVATION')) {
         return res.sendStatus(403);
       }
-
-      var tasks = [];
-      if (validateObservationId) {
-        tasks.push(function (done) {
-          /*
-          TODO: this is validating the id from body document, but should it
-          validate the id from the url path instead? the path id is the one
-          that is passed down to the update operation
-          */
-          new api.Observation().validateObservationId(req.param('id'), done);
-        });
-      }
-
-      tasks.push(function (done) {
-        Team.teamsForUserInEvent(req.user, req.event, function (err, teams) {
-          if (err) return next(err);
-
-          if (teams.length === 0) {
-            return res.status(403).send('Cannot submit an observation for an event that you are not part of.');
-          }
-
-          done();
-        });
-      });
-
-      async.series(tasks, next);
-    };
-  }
-
-  async function validateObservationUpdateAccess(req, res, next) {
-    if (access.userHasPermission(req.user, 'UPDATE_OBSERVATION_ALL')) {
-      return next();
-    }
-    if (access.userHasPermission(req.user, 'UPDATE_OBSERVATION_EVENT')) {
-      // Make sure I am part of this event
-      const hasPermission = await eventPermissions.userHasEventPermission(req.event, req.user.id, 'read')
-      if (hasPermission) {
-        return next();
+      const isParticipant = await eventPermissions.userIsParticipantInEvent(req.event, req.user.id)
+      if (isParticipant) {
+        return next()
       }
+      res.status(403).send('You must be an event participant to perform this operation.')
     }
-    res.sendStatus(403);
-  }
-
-  function validateAttachmentFile(req, res, next) {
-    FileType.fromFile(req.file.path).then(fileType => {
-      const attachment = req.observation.attachments.find(attachment => attachment._id.toString() === req.params.attachmentId);
-      const observationForm = req.observation.properties.forms.find(observationForm => {
-        return observationForm._id.toString() === attachment.observationFormId.toString()
-      });
-      if (!observationForm) {
-        return res.status(400).send('Attachment form not found');
-      }
-      const formDefinition = req.event.forms.find(form => form._id === observationForm.formId);
-      const fieldDefinition = formDefinition.fields.find(field => field.name === attachment.fieldName);
-      if (!fieldDefinition) {
-        return res.status(400).send('Attachment field not found');
-      }
-      if (!attachmentTypeIsValidForField(fieldDefinition, fileType.mime)) {
-        return res.status(400).send(`Invalid attachment '${attachment.name}', type must be one of ${fieldDefinition.allowedAttachmentTypes.join(' or ')}`);
-      }
-      next();
-    });
   }
 
   function authorizeEventAccess(collectionPermission, aclPermission) {
@@ -302,26 +239,6 @@ module.exports = function (app, security) {
     }
   );
 
-  app.get(
-    '/api/events/:eventId/observations/:observationIdInPath',
-    passport.authenticate('bearer'),
-    validateObservationReadAccess,
-    parseQueryParams,
-    function (req, res, next) {
-      const options = { fields: req.parameters.fields };
-      new api.Observation(req.event).getById(req.params.observationIdInPath, options, function (err, observation) {
-        if (err) {
-          return next(err);
-        }
-        if (!observation) {
-          return res.sendStatus(404);
-        }
-        const response = observationXform.transform(observation, transformOptions(req));
-        res.json(response);
-      });
-    }
-  );
-
   app.get(
     '/api/events/:eventId/observations',
     passport.authenticate('bearer'),
diff --git a/service/test/adapters/observations/adapters.observations.controllers.web.test.ts b/service/test/adapters/observations/adapters.observations.controllers.web.test.ts
index 2b4a7a061..11e43f0fc 100644
--- a/service/test/adapters/observations/adapters.observations.controllers.web.test.ts
+++ b/service/test/adapters/observations/adapters.observations.controllers.web.test.ts
@@ -166,6 +166,13 @@ describe('observations web controller', function () {
     })
   })
 
+  describe('GET /observations/{observationId}', function() {
+
+    it('has tests', async function() {
+      expect.fail('todo')
+    })
+  })
+
   describe('PUT /observations/{observationId}', function() {
 
     it('saves the observation for a mod request', async function() {
diff --git a/service/test/app/observations/app.observations.test.ts b/service/test/app/observations/app.observations.test.ts
index 8572c64c1..79931f3fb 100644
--- a/service/test/app/observations/app.observations.test.ts
+++ b/service/test/app/observations/app.observations.test.ts
@@ -8,7 +8,7 @@ import { addAttachment, Attachment, AttachmentContentPatchAttrs, AttachmentCreat
 import { permissionDenied, MageError, ErrPermissionDenied, ErrEntityNotFound, EntityNotFoundError, InvalidInputError, ErrInvalidInput, PermissionDeniedError, InfrastructureError, ErrInfrastructure } from '../../../lib/app.api/app.api.errors'
 import { FormFieldType } from '../../../lib/entities/events/entities.events.forms'
 import _ from 'lodash'
-import { User, UserId, UserRepository } from '../../../lib/entities/users/entities.users'
+import { User, UserIconType, UserId, UserRepository } from '../../../lib/entities/users/entities.users'
 import { pipeline, Readable } from 'stream'
 import util from 'util'
 import { BufferWriteable } from '../../utils'
@@ -343,7 +343,8 @@ describe('observations use case interactions', function() {
           { minDimension: 200, contentLocator: void(0), size: 2000, contentType: 'image/jpeg', width: 200, height: 300, name: 'rainbow@200.jpg' },
           { minDimension: 300, contentLocator: void(0), size: 3000, contentType: 'image/jpeg', width: 300, height: 400, name: 'rainbow@300.jpg' },
           { minDimension: 400, contentLocator: void(0), size: 4000, contentType: 'image/jpeg', width: 400, height: 500, name: 'rainbow@400.jpg' },
-        ]
+        ],
+        url: '/remove/me'
       }
 
       expect(api.exoAttachmentForThumbnail(0, att)).to.deep.equal({
@@ -497,6 +498,9 @@ describe('observations use case interactions', function() {
         phones: [],
         roleId: uniqid(),
         username: 'populate.me',
+        avatar: {},
+        icon: { type: UserIconType.None },
+        recentEventIds: []
       }
       const req: api.SaveObservationRequest = {
         context,
@@ -530,6 +534,9 @@ describe('observations use case interactions', function() {
         phones: [],
         roleId: uniqid(),
         username: 'populate.me',
+        recentEventIds: [],
+        avatar: {},
+        icon: { type: UserIconType.None }
       }
       const req: api.SaveObservationRequest = {
         context,
@@ -567,6 +574,9 @@ describe('observations use case interactions', function() {
         phones: [],
         roleId: uniqid(),
         username: 'user1',
+        recentEventIds: [],
+        avatar: {},
+        icon: { type: UserIconType.None }
       }
       const importantFlagger: User = {
         id: uniqid(),
@@ -579,6 +589,9 @@ describe('observations use case interactions', function() {
         phones: [],
         roleId: uniqid(),
         username: 'user2',
+        recentEventIds: [],
+        avatar: {},
+        icon: { type: UserIconType.None }
       }
       const req: api.SaveObservationRequest = {
         context,
@@ -1530,6 +1543,21 @@ describe('observations use case interactions', function() {
     })
   })
 
+  describe('reading observations', function() {
+
+    describe('reading by id', function() {
+      it('has tests', async function() {
+        expect.fail('todo')
+      })
+    })
+
+    describe('searching', function() {
+      it('has tests', async function() {
+        expect.fail('todo')
+      })
+    })
+  })
+
   describe('saving attachment content', function() {
 
     let storeAttachmentContent: api.StoreAttachmentContent

From 84ba2e517b436d20217f54338b456c7ab1cb3c12 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sun, 4 Aug 2024 10:06:09 -0600
Subject: [PATCH 035/183] fix!(service): allow plugin state repo put operation
 to accept and return null

---
 service/src/plugins.api/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/plugins.api/index.ts b/service/src/plugins.api/index.ts
index c8dfeb873..096a2e68f 100644
--- a/service/src/plugins.api/index.ts
+++ b/service/src/plugins.api/index.ts
@@ -54,7 +54,7 @@ import { EnsureJson } from '../entities/entities.json_types'
  * state the plugin requires.
  */
 export interface PluginStateRepository<State extends object> {
-  put(state: EnsureJson<State>): Promise<EnsureJson<State>>
+  put(state: EnsureJson<State> | null): Promise<EnsureJson<State> | null>
   patch(state: Partial<EnsureJson<State>>): Promise<EnsureJson<State>>
   get(): Promise<EnsureJson<State> | null>
 }

From 3172492ab3b57fd33220c98c2f3de276e26bab53 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 5 Aug 2024 10:12:59 -0600
Subject: [PATCH 036/183] chore(service): hook up read observation operation in
 main app module

---
 .../adapters.observations.controllers.web.ts     | 16 +++++++++++++---
 .../app.api/observations/app.api.observations.ts |  6 +++++-
 service/src/app.ts                               |  4 +++-
 3 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/service/src/adapters/observations/adapters.observations.controllers.web.ts b/service/src/adapters/observations/adapters.observations.controllers.web.ts
index 1bd89457c..a2f83fbc5 100644
--- a/service/src/adapters/observations/adapters.observations.controllers.web.ts
+++ b/service/src/adapters/observations/adapters.observations.controllers.web.ts
@@ -1,6 +1,6 @@
 import express from 'express'
 import { compatibilityMageAppErrorHandler } from '../adapters.controllers.web'
-import { AllocateObservationId, ExoAttachment, ExoIncomingAttachmentContent, ExoObservation, ExoObservationMod, ObservationRequest, ReadAttachmentContent, ReadAttachmentContentRequest, SaveObservation, SaveObservationRequest, StoreAttachmentContent, StoreAttachmentContentRequest } from '../../app.api/observations/app.api.observations'
+import { AllocateObservationId, ExoAttachment, ExoIncomingAttachmentContent, ExoObservation, ObservationRequest, ReadAttachmentContent, ReadAttachmentContentRequest, ReadObservation, SaveObservation, SaveObservationRequest, StoreAttachmentContent, StoreAttachmentContentRequest } from '../../app.api/observations/app.api.observations'
 import { AttachmentStore, EventScopedObservationRepository, ObservationState } from '../../entities/observations/entities.observations'
 import { MageEvent, MageEventId } from '../../entities/events/entities.events'
 import busboy from 'busboy'
@@ -16,6 +16,7 @@ declare module 'express-serve-static-core' {
 export interface ObservationAppLayer {
   allocateObservationId: AllocateObservationId
   saveObservation: SaveObservation
+  readObservation: ReadObservation
   storeAttachmentContent: StoreAttachmentContent
   readAttachmentContent: ReadAttachmentContent
 }
@@ -70,8 +71,8 @@ export function ObservationRoutes(app: ObservationAppLayer, attachmentStore: Att
       },
       async (req, res, next) => {
         const afterUploadStreamEvent = 'afterUploadStream'
-        const sendInvalidRequestStructure = () => next(invalidInput(`request must contain only one file part named 'attachment'`))
-        const afterUploadStream = (finishResponse: () => void) => {
+        const sendInvalidRequestStructure = (): void => next(invalidInput(`request must contain only one file part named 'attachment'`))
+        const afterUploadStream = (finishResponse: () => void): void => {
           if (req.attachmentUpload?.listenerCount(afterUploadStreamEvent)) {
             return
           }
@@ -190,6 +191,15 @@ export function ObservationRoutes(app: ObservationAppLayer, attachmentStore: Att
       }
       next(appRes.error)
     })
+    .get(async (req, res, next) => {
+      const observationId = req.params.observationId
+      const appReq = createAppRequest(req, { observationId })
+      const appRes = await app.readObservation(appReq)
+      if (appRes.success) {
+        return res.json(jsonForObservation(appRes.success, qualifiedBaseUrl(req)))
+      }
+      next(appRes.error)
+    })
 
   return routes.use(compatibilityMageAppErrorHandler)
 }
diff --git a/service/src/app.api/observations/app.api.observations.ts b/service/src/app.api/observations/app.api.observations.ts
index b398304b9..8fb7691ff 100644
--- a/service/src/app.api/observations/app.api.observations.ts
+++ b/service/src/app.api/observations/app.api.observations.ts
@@ -15,7 +15,11 @@ export interface ObservationRequestContext<Principal = unknown> extends AppReque
    * probably be a user-device pair, eventually.
    */
   userId: UserId
-  deviceId: string
+  /**
+   * TODO: device id: The `None` device provisioning strategy returns a dummy "device"
+   * object that has no ID.
+   */
+  deviceId?: string
   observationRepository: EventScopedObservationRepository
 }
 export interface ObservationRequest<Principal = unknown> extends AppRequest<Principal, ObservationRequestContext<Principal>> {}
diff --git a/service/src/app.ts b/service/src/app.ts
index 9d6413ad3..ee6c09ce9 100644
--- a/service/src/app.ts
+++ b/service/src/app.ts
@@ -140,7 +140,7 @@ export const boot = async function(config: BootConfig): Promise<MageService> {
   const appLayer = await initAppLayer(repos)
   const { webController, addAuthenticatedPluginRoutes } = await initWebLayer(repos, appLayer, config.plugins?.webUIPlugins || [])
   const routesForPluginId: { [pluginId: string]: WebRoutesHooks['webRoutes'] } = {}
-  const collectPluginRoutesToSort = (pluginId: string, initPluginRoutes: WebRoutesHooks['webRoutes']) => {
+  const collectPluginRoutesToSort = (pluginId: string, initPluginRoutes: WebRoutesHooks['webRoutes']): void => {
     routesForPluginId[pluginId] = initPluginRoutes
   }
   const globalScopeServices = new Map<InjectionToken<any>, any>([
@@ -250,6 +250,7 @@ type AppLayer = {
   observations: {
     allocateObservationId: observationsApi.AllocateObservationId
     saveObservation: observationsApi.SaveObservation
+    readObservation: observationsApi.ReadObservation
     storeAttachmentContent: observationsApi.StoreAttachmentContent
     readAttachmentContent: observationsApi.ReadAttachmentContent
   },
@@ -462,6 +463,7 @@ async function initObservationsAppLayer(repos: Repositories): Promise<AppLayer['
   return {
     allocateObservationId: observationsImpl.AllocateObservationId(obsPermissionsService),
     saveObservation: observationsImpl.SaveObservation(obsPermissionsService, repos.users.userRepo),
+    readObservation: observationsImpl.ReadObservation(obsPermissionsService),
     storeAttachmentContent: observationsImpl.StoreAttachmentContent(obsPermissionsService, repos.observations.attachmentStore),
     readAttachmentContent: observationsImpl.ReadAttachmentContent(obsPermissionsService, repos.observations.attachmentStore)
   }

From 2ca4f771aba305d2087ff8f5ddc9c99d3067002f Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 5 Aug 2024 12:52:22 -0600
Subject: [PATCH 037/183] docs(service): add deprecation docs for absolute urls

---
 .../observations/adapters.observations.controllers.web.ts | 7 +++++++
 .../observations/adapters.observations.db.mongoose.ts     | 8 +++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/service/src/adapters/observations/adapters.observations.controllers.web.ts b/service/src/adapters/observations/adapters.observations.controllers.web.ts
index a2f83fbc5..016552375 100644
--- a/service/src/adapters/observations/adapters.observations.controllers.web.ts
+++ b/service/src/adapters/observations/adapters.observations.controllers.web.ts
@@ -218,6 +218,13 @@ export type WebAttachment = ExoAttachment & {
   url?: string
 }
 
+/**
+ * Map the given observation to a {@link WebObservation} JSON object which has extra URL
+ * entries based on the current base URL of the web app.
+ * @deprecated TODO: abs url: Stop using absolute URLs with FQDN.  Clients should constuct
+ * requests based on the ReST API definition or using relative URLs appended to the base
+ * URL of the server.
+ */
 export function jsonForObservation(o: ExoObservation, baseUrl: string): WebObservation {
   const obsUrl = `${baseUrl}/${o.id}`
   return {
diff --git a/service/src/adapters/observations/adapters.observations.db.mongoose.ts b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
index e710ad584..dbc484b92 100644
--- a/service/src/adapters/observations/adapters.observations.db.mongoose.ts
+++ b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
@@ -117,6 +117,12 @@ function hasOwnProperty(wut: any, property: PropertyKey): boolean {
   return Object.prototype.hasOwnProperty.call(wut, property)
 }
 
+/**
+ * @deprecated TODO: obs types: This exists for backward compatibility, but should be viable to remove when all
+ * observation routes that return observation json transition to DI architecture in the adapter layer, as the newer
+ * `WebObservation` type mapping handles adding the `url` key to observation documents in the observation
+ * [web controller](./adapters.observations.controllers.web.ts).
+ */
 function transformObservationModelInstance(modelInstance: mongoose.HydratedDocument<ObservationDocument, ObservationSubdocuments> | mongoose.HydratedDocument<ObservationIdDocument>, result: any, options: ObservationTransformOptions): ObservationAttrs {
   result.id = modelInstance._id.toHexString()
   delete result._id
@@ -172,7 +178,7 @@ function transformObservationModelInstance(modelInstance: mongoose.HydratedDocum
   return result
 }
 
-ObservationIdSchema.set("toJSON", { transform: transformObservationModelInstance })
+ObservationIdSchema.set('toJSON', { transform: transformObservationModelInstance })
 
 export const StateSchema = new Schema({
   name: { type: String, required: true },

From de0c152a7dfa9f66f2869dc77200f1f9d929ce88 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 5 Aug 2024 21:16:17 -0600
Subject: [PATCH 038/183] style(service): update syntax in geoJsonFormat module

---
 service/src/format/geoJsonFormat.js | 83 ++++++++++++++---------------
 1 file changed, 41 insertions(+), 42 deletions(-)

diff --git a/service/src/format/geoJsonFormat.js b/service/src/format/geoJsonFormat.js
index dc63e5b54..ea45cbc81 100644
--- a/service/src/format/geoJsonFormat.js
+++ b/service/src/format/geoJsonFormat.js
@@ -1,50 +1,51 @@
-var parseEnvelope = function(text) {
-  var bbox = JSON.parse(text);
-  if (bbox.length !== 4) {
+function parseEnvelope(text) {
+  const coords = JSON.parse(text);
+  if (coords.length !== 4) {
     throw new Error("Invalid geometry: " + text);
   }
 
-  var xmin = parseFloat(bbox[0]);
-  var ymin = parseFloat(bbox[1]);
-  var xmax = parseFloat(bbox[2]);
-  var ymax = parseFloat(bbox[3]);
+  let xmin = parseFloat(coords[0]);
+  let ymin = parseFloat(coords[1]);
+  let xmax = parseFloat(coords[2]);
+  let ymax = parseFloat(coords[3]);
 
+  // TODO: is this the right thing to do?
   xmin = xmin < -180 ? -180 : xmin;
   ymin = ymin < -90 ? -90 : ymin;
   xmax = xmax > 180 ? 180 : xmax;
   ymax = ymax > 90 ? 90 : ymax;
 
-  bbox = {xmin: xmin, ymin: ymin, xmax: xmax, ymax: ymax };
+  const bbox = { xmin, ymin, xmax, ymax };
 
-  var geometries = [];
-
-  // TODO hack until mongo fixes queries for more than
+  // TODO: hack until mongo fixes queries for more than
   // 180 degrees longitude.  Create 2 geometries if we cross
   // the prime meridian
   if (bbox.xmax > 0 && bbox.xmin < 0) {
-    geometries.push({
-      type: 'Polygon',
-      coordinates: [ [
-        [bbox.xmin, bbox.ymin],
-        [0, bbox.ymin],
-        [0, bbox.ymax],
-        [bbox.xmin, bbox.ymax],
-        [bbox.xmin, bbox.ymin]
-      ] ]
-    });
-
-    geometries.push({
-      type: 'Polygon',
-      coordinates: [ [
-        [0, bbox.ymin],
-        [bbox.xmax, bbox.ymin],
-        [bbox.xmax, bbox.ymax],
-        [0, bbox.ymax],
-        [0, bbox.ymin]
-      ] ]
-    });
-  } else {
-    geometries.push({
+    return [
+      {
+        type: 'Polygon',
+        coordinates: [ [
+          [bbox.xmin, bbox.ymin],
+          [0, bbox.ymin],
+          [0, bbox.ymax],
+          [bbox.xmin, bbox.ymax],
+          [bbox.xmin, bbox.ymin]
+        ] ]
+      },
+      {
+        type: 'Polygon',
+        coordinates: [ [
+          [0, bbox.ymin],
+          [bbox.xmax, bbox.ymin],
+          [bbox.xmax, bbox.ymax],
+          [0, bbox.ymax],
+          [0, bbox.ymin]
+        ] ]
+      }
+    ]
+  }
+  return [
+    {
       type: 'Polygon',
       coordinates: [ [
         [bbox.xmin, bbox.ymin],
@@ -53,19 +54,17 @@ var parseEnvelope = function(text) {
         [bbox.xmin, bbox.ymax],
         [bbox.xmin, bbox.ymin]
       ] ]
-    });
-  }
-
-  return geometries;
-};
+    }
+  ]
+}
 
-var parseGeometry = function(type, text) {
+function parseGeometry(type, text) {
   switch (type) {
   case 'bbox':
     return parseEnvelope(text);
   default:
-    return [JSON.parse(text)];
+    return [ JSON.parse(text) ];
   }
-};
+}
 
 exports.parse = parseGeometry;

From fc2694cdde0de9eeb84e31139623dfc349df3890 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 5 Aug 2024 21:20:39 -0600
Subject: [PATCH 039/183] style(service): unused import

---
 .../observations/adapters.observations.dto.ecma404-json.ts       | 1 -
 1 file changed, 1 deletion(-)

diff --git a/service/src/adapters/observations/adapters.observations.dto.ecma404-json.ts b/service/src/adapters/observations/adapters.observations.dto.ecma404-json.ts
index ac6650df4..8dfe5fbbd 100644
--- a/service/src/adapters/observations/adapters.observations.dto.ecma404-json.ts
+++ b/service/src/adapters/observations/adapters.observations.dto.ecma404-json.ts
@@ -3,7 +3,6 @@ import moment from 'moment'
 import { invalidInput, InvalidInputError } from '../../app.api/app.api.errors'
 import { ExoObservationMod } from '../../app.api/observations/app.api.observations'
 import { Json } from '../../entities/entities.json_types'
-import { ObservationId } from '../../entities/observations/entities.observations'
 
 /*
  * NOTE: This file is named ecma404-json to avoid any potential problems with

From 526a88de4465b9481e159cb2976a274a9754b57b Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 5 Aug 2024 22:23:24 -0600
Subject: [PATCH 040/183] fix(service): observation state name was supposed to
 be `archive`, not `archived`

---
 .../entities/observations/entities.observations.ts    |  4 ++--
 .../adapters.observations.db.mongoose.test.ts         | 11 +++++------
 .../test/app/observations/app.observations.test.ts    |  4 ++--
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/service/src/entities/observations/entities.observations.ts b/service/src/entities/observations/entities.observations.ts
index 505a89a99..dcadbba4e 100644
--- a/service/src/entities/observations/entities.observations.ts
+++ b/service/src/entities/observations/entities.observations.ts
@@ -71,7 +71,7 @@ export interface ObservationImportantFlag {
 
 export interface ObservationState {
   id: string | PendingEntityId
-  name: 'active' | 'archived'
+  name: 'active' | 'archive'
   userId?: UserId | undefined
   /**
    * @deprecated TODO: confine URLs to the web layer
@@ -668,7 +668,7 @@ export function removeFormEntry(observation: Observation, formEntryId: FormEntry
   return Observation.assignTo(observation, mod) as Observation
 }
 
-export type AttachmentCreateAttrs = Omit<Attachment, 'id' | 'observationFormId' | 'fieldName' | 'lastModified'>
+export type AttachmentCreateAttrs = Omit<Attachment, 'id' | 'observationFormId' | 'fieldName' | 'lastModified' | 'url'>
 export type AttachmentPatchAttrs = Partial<AttachmentCreateAttrs>
 export type AttachmentContentPatchAttrs = Required<Pick<Attachment, 'contentLocator' | 'size'>>
 export type ThumbnailContentPatchAttrs = Required<Pick<Thumbnail, 'contentLocator' | 'size'>> & Thumbnail
diff --git a/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts b/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts
index 6ccdd8558..091ccf479 100644
--- a/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts
+++ b/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts
@@ -3,13 +3,12 @@ import { expect } from 'chai'
 import mongoose from 'mongoose'
 import _ from 'lodash'
 import { MongooseMageEventRepository } from '../../../lib/adapters/events/adapters.events.db.mongoose'
-import { MongooseObservationRepository } from '../../../lib/adapters/observations/adapters.observations.db.mongoose'
+import { MongooseObservationRepository, ObservationModel } from '../../../lib/adapters/observations/adapters.observations.db.mongoose'
 import * as legacy from '../../../lib/models/observation'
 import * as legacyEvent from '../../../lib/models/event'
 import { MageEventDocument } from '../../../src/models/event'
 
 import { MageEvent, MageEventAttrs, MageEventCreateAttrs, MageEventId } from '../../../lib/entities/events/entities.events'
-import { ObservationDocument, ObservationModel } from '../../../src/models/observation'
 import { ObservationAttrs, ObservationId, Observation, ObservationRepositoryError, ObservationRepositoryErrorCode, copyObservationAttrs, AttachmentContentPatchAttrs, copyAttachmentAttrs, AttachmentNotFoundError, AttachmentPatchAttrs, removeAttachment, validationResultMessage, ObservationDomainEventType, ObservationEmitted, PendingObservationDomainEvent, AttachmentsRemovedDomainEvent } from '../../../lib/entities/observations/entities.observations'
 import { AttachmentPresentationType, FormFieldType, Form, AttachmentMediaTypes } from '../../../lib/entities/events/entities.events.forms'
 import util from 'util'
@@ -153,7 +152,7 @@ describe('mongoose observation repository', function() {
 
       expect(id).to.be.a.string
       expect(id).to.not.be.empty
-      expect(parsed.equals(found?._id)).to.be.true
+      expect(parsed.equals(found?._id || '')).to.be.true
       expect(idCount).to.equal(1)
     })
   })
@@ -247,7 +246,7 @@ describe('mongoose observation repository', function() {
           },
           {
             id: (new mongoose.Types.ObjectId()).toHexString(),
-            name: 'archived',
+            name: 'archive',
             userId: undefined
           }
         ]
@@ -326,7 +325,7 @@ describe('mongoose observation repository', function() {
           coordinates: [ 12, 34 ]
         }
         putAttrs.states = [
-          { name: 'archived', id: PendingEntityId }
+          { name: 'archive', id: PendingEntityId }
         ]
         putAttrs.properties.forms = [
           {
@@ -412,7 +411,7 @@ describe('mongoose observation repository', function() {
       state1Stub.states = [
         {
           id: PendingEntityId,
-          name: 'archived',
+          name: 'archive',
           userId: (new mongoose.Types.ObjectId()).toHexString()
         }
       ]
diff --git a/service/test/app/observations/app.observations.test.ts b/service/test/app/observations/app.observations.test.ts
index 79931f3fb..7a3ffe8a4 100644
--- a/service/test/app/observations/app.observations.test.ts
+++ b/service/test/app/observations/app.observations.test.ts
@@ -290,7 +290,7 @@ describe('observations use case interactions', function() {
       expect(exo.state).to.be.undefined
 
       const states: ObservationState[] = [
-        { id: uniqid(), name: 'archived', userId: uniqid() },
+        { id: uniqid(), name: 'archive', userId: uniqid() },
         { id: uniqid(), name: 'active', userId: uniqid() }
       ]
       from.states = states.map(copyObservationStateAttrs)
@@ -1418,7 +1418,7 @@ describe('observations use case interactions', function() {
           id: uniqid(),
           states: [
             { id: uniqid(), name: 'active', userId: uniqid() },
-            { id: uniqid(), name: 'archived', userId: uniqid() }
+            { id: uniqid(), name: 'archive', userId: uniqid() }
           ]
         }, mageEvent) as Observation
         const obsAfter = Observation.assignTo(obsBefore, {

From a2474e524e6af85aed790aa90bad1fc385d81edc Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 6 Aug 2024 06:58:24 -0600
Subject: [PATCH 041/183] chore(service): remove unreferenced legacy attachment
 api code

---
 service/src/api/attachment.js | 86 ++---------------------------------
 1 file changed, 5 insertions(+), 81 deletions(-)

diff --git a/service/src/api/attachment.js b/service/src/api/attachment.js
index 010688d00..57bc95fd9 100644
--- a/service/src/api/attachment.js
+++ b/service/src/api/attachment.js
@@ -1,91 +1,15 @@
-const ObservationModel = require('../models/observation')
-  , log = require('winston')
-  , path = require('path')
-  , fs = require('fs-extra')
-  , environment = require('../environment/env');
+const log = require('winston')
+const path = require('path')
+const fs = require('fs-extra')
+const environment = require('../environment/env')
 
-const attachmentBase = environment.attachmentBaseDirectory;
-
-const createAttachmentPath = function(event) {
-  const now = new Date();
-  return path.join(
-    event.collectionName,
-    now.getFullYear().toString(),
-    (now.getMonth() + 1).toString(),
-    now.getDate().toString()
-  );
-};
+const attachmentBase = environment.attachmentBaseDirectory
 
 function Attachment(event, observation) {
   this._event = event;
   this._observation = observation;
 }
 
-Attachment.prototype.getById = function(attachmentId, options, callback) {
-  const size = options.size ? Number(options.size) : null;
-
-  ObservationModel.getAttachment(this._event, this._observation._id, attachmentId, function(err, attachment) {
-    if (!attachment) return callback(err);
-
-    if (size) {
-      attachment.thumbnails.forEach(function(thumbnail) {
-        if ((thumbnail.minDimension < attachment.height || !attachment.height) &&
-          (thumbnail.minDimension < attachment.width || !attachment.width) &&
-          (thumbnail.minDimension >= size)) {
-          attachment = thumbnail;
-        }
-      });
-    }
-
-    if (attachment && attachment.relativePath) attachment.path = path.join(attachmentBase, attachment.relativePath);
-
-    callback(null, attachment);
-  });
-};
-
-Attachment.prototype.update = function(attachmentId, attachment, callback) {
-  const relativePath = createAttachmentPath(this._event);
-  // move file upload to its new home
-  const dir = path.join(attachmentBase, relativePath);
-  fs.mkdirp(dir, err => {
-    if (err) return callback(err);
-
-    const fileName = path.basename(attachment.path);
-    attachment.relativePath = path.join(relativePath, fileName);
-    const file = path.join(attachmentBase, attachment.relativePath);
-
-    fs.move(attachment.path, file, err => {
-      if (err) return callback(err);
-
-      ObservationModel.addAttachment(this._event, this._observation._id, attachmentId, attachment, (err, newAttachment) => {
-        // TODO: now defunct after removing legacy attachment events module
-        // if (!err && newAttachment) {
-        //   EventEmitter.emit(AttachmentEvents.events.add, newAttachment.toObject(), this._observation, this._event);
-        // }
-        callback(err, newAttachment);
-      });
-    });
-  });
-};
-
-Attachment.prototype.delete = function(attachmentId, callback) {
-  const attachment = this._observation.attachments.find(attachment => attachment._id.toString() === attachmentId);
-  ObservationModel.removeAttachment(this._event, this._observation._id, attachmentId, err => {
-    if (err) return callback(err);
-
-    if (attachment && attachment.relativePath) {
-      const file = path.join(attachmentBase, attachment.relativePath);
-      fs.remove(file, err => {
-        if (err) {
-          log.error('Could not remove attachment file ' + file + '.', err);
-        }
-      });
-    }
-
-    callback();
-  });
-};
-
 /**
  * TODO: this no longer works with the directory scheme `FileSystemAttachmentStore` uses.
  */

From 31f79fbe9d4a73b5dd813a479011f0cc5e8185d7 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 7 Aug 2024 15:13:05 -0600
Subject: [PATCH 042/183] refactor(service): migrate read observations rest api
 to DI adapter layer

---
 .../adapters.observations.controllers.web.ts  | 130 +++++++++++++++++-
 .../adapters.observations.db.mongoose.ts      |   5 +-
 .../observations/app.api.observations.ts      |  23 ++++
 .../observations/entities.observations.ts     |  19 ++-
 service/src/routes/observations.js            | 102 --------------
 ...pters.observations.controllers.web.test.ts |  39 ++++++
 .../app/observations/app.observations.test.ts |   4 +-
 7 files changed, 213 insertions(+), 109 deletions(-)

diff --git a/service/src/adapters/observations/adapters.observations.controllers.web.ts b/service/src/adapters/observations/adapters.observations.controllers.web.ts
index 016552375..2b0856164 100644
--- a/service/src/adapters/observations/adapters.observations.controllers.web.ts
+++ b/service/src/adapters/observations/adapters.observations.controllers.web.ts
@@ -1,11 +1,12 @@
 import express from 'express'
 import { compatibilityMageAppErrorHandler } from '../adapters.controllers.web'
-import { AllocateObservationId, ExoAttachment, ExoIncomingAttachmentContent, ExoObservation, ObservationRequest, ReadAttachmentContent, ReadAttachmentContentRequest, ReadObservation, SaveObservation, SaveObservationRequest, StoreAttachmentContent, StoreAttachmentContentRequest } from '../../app.api/observations/app.api.observations'
-import { AttachmentStore, EventScopedObservationRepository, ObservationState } from '../../entities/observations/entities.observations'
+import { AllocateObservationId, ExoAttachment, ExoIncomingAttachmentContent, ExoObservation, ObservationRequest, ReadAttachmentContent, ReadAttachmentContentRequest, ReadObservation, ReadObservations, ReadObservationsRequest, SaveObservation, SaveObservationRequest, StoreAttachmentContent, StoreAttachmentContentRequest } from '../../app.api/observations/app.api.observations'
+import { AttachmentStore, EventScopedObservationRepository, ObservationState, ObservationStateName } from '../../entities/observations/entities.observations'
 import { MageEvent, MageEventId } from '../../entities/events/entities.events'
-import busboy from 'busboy'
 import { invalidInput } from '../../app.api/app.api.errors'
 import { exoObservationModFromJson } from './adapters.observations.dto.ecma404-json'
+import busboy from 'busboy'
+import moment from 'moment'
 
 declare module 'express-serve-static-core' {
   interface Request {
@@ -16,6 +17,7 @@ declare module 'express-serve-static-core' {
 export interface ObservationAppLayer {
   allocateObservationId: AllocateObservationId
   saveObservation: SaveObservation
+  readObservations: ReadObservations
   readObservation: ReadObservation
   storeAttachmentContent: StoreAttachmentContent
   readAttachmentContent: ReadAttachmentContent
@@ -201,9 +203,131 @@ export function ObservationRoutes(app: ObservationAppLayer, attachmentStore: Att
       next(appRes.error)
     })
 
+  routes.route('/')
+    .get(async (req, res, next) => {
+      const readSpec: Pick<ReadObservationsRequest, 'filter' | 'sort' | 'populate'> = parseObservationQueryParams(req)
+      const appReq = createAppRequest(req, readSpec)
+      const appRes = await app.readObservations(appReq)
+      if (appRes.success) {
+        return res.json(appRes.success.map(x => jsonForObservation(x, qualifiedBaseUrl(req))))
+      }
+      next(appRes.error)
+    })
+
   return routes.use(compatibilityMageAppErrorHandler)
 }
 
+/**
+ * Attempt to parse the given string to an array of numbers that represents a
+ * bounding box of the form [ xMin, yMin, xMax, yMax ].  This does not validate
+ * lat/lon bounds, only array length and number type.  The string can be a
+ * JSON string number array (deprecated), e.g., `'[ 1, 2, 3, 4 ]'`, or a comma-
+ * separated list, e.g., `'1,2,3,4'`.
+ */
+function parseBBox(maybeBBoxString: any): number[] | null {
+  if (typeof maybeBBoxString !== 'string') {
+    return null
+  }
+  let parsed: number[] = []
+  try {
+    // TODO: move this geometryFormat.parse() call down to mongodb repository
+    // filter.geometries = geometryFormat.parse('bbox', bbox)
+    // TODO: would be better not to embed json strings in query parameters; use csv instead
+    parsed = JSON.parse(maybeBBoxString)
+    if (!Array.isArray(parsed)) {
+      return null
+    }
+    return null
+  }
+  catch(err) {
+    console.debug('invalid json string from query parameter `bbox`', maybeBBoxString, err)
+  }
+  // try csv instead of json
+  // TODO: this should be the only supported format
+  if (!parsed) {
+    parsed = maybeBBoxString.split(',').map(parseFloat)
+  }
+  if (parsed.length !== 4 && parsed.some(x => typeof x !== 'number' || isNaN(x))) {
+    return null
+  }
+  return parsed
+}
+
+/**
+ * Parse {@link ObservationStateName} strings from the given input string.  This expects the input string to be
+ * comma-separated values with no spaces.  Only parse the first N state names, where N is number of valid state names.
+ * Return null if the input is not a string or contains no valid state names.
+ */
+function parseStatesParam(maybeStatesString: any): ObservationStateName[] | null {
+  if (typeof maybeStatesString !== 'string') {
+    return null
+  }
+  const allStateNames = Object.values(ObservationStateName)
+  const states = maybeStatesString.split(',', allStateNames.length).reduce((states: Set<ObservationStateName>, stateName: any) => {
+    if (allStateNames.includes(stateName) && !states.has(stateName)) {
+      return states.add(stateName)
+    }
+    return states
+  }, new Set<ObservationStateName>())
+  return states.size > 0 ? Array.from(states.values()) : null
+}
+
+const allowedSortFields = {
+  lastModified: true,
+  timestamp: true,
+} as Record<string, true>
+
+/**
+ * Parse a sort field specification of the form `field+order`, where `field` is the name of an observation field,
+ * and `order` is `desc`, `-`, or `-1`, to indicate a descending sort.  The default sort order is ascending.  Only the
+ * first valid sort field is used
+ */
+function parseSortParam(maybeSortString: any): ReadObservationsRequest['sort'] | null {
+  if (typeof maybeSortString !== 'string') {
+    return null
+  }
+  const sort = maybeSortString.split(',').reduce<{ field: string, order: 1 | -1 }[]>((sort, sortFieldSpec) => {
+    const [ name, orderString ] = sortFieldSpec.split('+')
+    const order = orderString?.toLowerCase() === 'desc' || orderString === '-' || orderString === '-1' ? -1 : 1
+    if (allowedSortFields[name] === true) {
+      return [ ...sort, { field: name, order } ]
+    }
+    return sort
+  }, [] as { field: string, order: 1 | -1 }[])[0]
+  return sort || null
+}
+
+function parseObservationQueryParams(req: express.Request): Pick<ReadObservationsRequest, 'filter' | 'sort' | 'populate'> {
+  const filter: ReadObservationsRequest['filter'] = {}
+  const startDate = req.query.startDate
+  if (startDate) {
+    filter.lastModifiedAfter = moment(String(startDate)).utc().toDate()
+  }
+  const endDate = req.query.endDate
+  if (endDate) {
+    filter.lastModifiedBefore = moment(String(endDate)).utc().toDate()
+  }
+  const observationStartDate = req.query.observationStartDate
+  if (observationStartDate) {
+    filter.timestampAfter = moment(String(observationStartDate)).utc().toDate()
+  }
+  const observationEndDate = req.query.observationEndDate
+  if (observationEndDate) {
+    filter.timestampBefore = moment(String(observationEndDate)).utc().toDate()
+  }
+  const bboxParam = parseBBox(req.query.bbox)
+  if (!bboxParam) {
+    console.warn('invalid bbox query parameter', req.query.bbox)
+  }
+  const states = parseStatesParam(req.query.states)
+  if (states) {
+    filter.states = states
+  }
+  const sort = parseSortParam(req.query.sort) || void(0)
+  const populate = req.query.populate === 'true'
+  return { filter, sort, populate }
+}
+
 export type WebObservation = Omit<ExoObservation, 'attachments' | 'state'> & {
   url: string
   state?: WebObservationState
diff --git a/service/src/adapters/observations/adapters.observations.db.mongoose.ts b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
index dbc484b92..5a6324a9e 100644
--- a/service/src/adapters/observations/adapters.observations.db.mongoose.ts
+++ b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
@@ -183,7 +183,7 @@ ObservationIdSchema.set('toJSON', { transform: transformObservationModelInstance
 export const StateSchema = new Schema({
   name: { type: String, required: true },
   userId: { type: Schema.Types.ObjectId, ref: 'User' }
-});
+})
 
 export const ThumbnailSchema = new Schema(
   {
@@ -252,6 +252,9 @@ ObservationSchema.index({ 'attachments.oriented': 1 })
 ObservationSchema.index({ 'attachments.contentType': 1 })
 ObservationSchema.index({ 'attachments.thumbnails.minDimension': 1 })
 
+/**
+ * TODO: add support for mongoose `lean()` queries
+ */
 export class MongooseObservationRepository extends BaseMongooseRepository<ObservationDocument, ObservationModel, ObservationAttrs> implements EventScopedObservationRepository {
 
   constructor(model: ObservationModel, readonly idModel: ObservationIdModel, readonly eventScope: MageEventId, readonly eventLookup: (eventId: MageEventId) => Promise<MageEvent | null>, readonly domainEvents: EventEmitter) {
diff --git a/service/src/app.api/observations/app.api.observations.ts b/service/src/app.api/observations/app.api.observations.ts
index 8fb7691ff..bf6871627 100644
--- a/service/src/app.api/observations/app.api.observations.ts
+++ b/service/src/app.api/observations/app.api.observations.ts
@@ -43,6 +43,29 @@ export interface ReadObservation {
   (req: ReadObservationRequest): Promise<AppResponse<ExoObservation, PermissionDeniedError | EntityNotFoundError>>
 }
 
+export interface ReadObservationsRequest extends ObservationRequest {
+  filter: {
+    lastModifiedAfter?: Date | undefined,
+    lastModifiedBefore?: Date | undefined,
+    timestampAfter?: Date | undefined,
+    timestampBefore?: Date | undefined,
+    bbox?: GeoJSON.BBox,
+    states?: ObservationState['name'][] | undefined,
+  },
+  sort?: {
+    field: string,
+    order?: 1 | -1,
+  },
+  /**
+   * If `true`, populate the user names for the observation {@link ObservationAttrs.userId creator} and
+   * {@link ObservationImportantFlag.userId important flag}.
+   */
+  populate?: boolean | undefined
+}
+export interface ReadObservations {
+  (req: ReadObservationsRequest): Promise<AppResponse<ExoObservation[], PermissionDeniedError | InvalidInputError>>
+}
+
 export interface StoreAttachmentContent {
   (req: StoreAttachmentContentRequest): Promise<AppResponse<ExoObservation, PermissionDeniedError | EntityNotFoundError | InvalidInputError | InfrastructureError>>
 }
diff --git a/service/src/entities/observations/entities.observations.ts b/service/src/entities/observations/entities.observations.ts
index dcadbba4e..00d5c5f86 100644
--- a/service/src/entities/observations/entities.observations.ts
+++ b/service/src/entities/observations/entities.observations.ts
@@ -16,6 +16,9 @@ export interface ObservationAttrs extends Feature<Geometry, ObservationFeaturePr
   // TODO: should be a strongly typed id-type
   deviceId?: string
   createdAt: Date
+  /**
+   * The last modified timestamp is the time of the latest edit of the observation's data.
+   */
   lastModified: Date
   attachments: readonly Attachment[]
   important?: Readonly<ObservationImportantFlag> | undefined
@@ -69,9 +72,23 @@ export interface ObservationImportantFlag {
   description?: string
 }
 
+export enum ObservationStateName {
+  Active = 'active',
+  /**
+   * This state essentially marks the observation as deleted.  The mobile apps use this so the server still returns
+   * deleted observations in queries and the mobile apps can delete their local records, or at least mark them deleted
+   * and hide them from view.
+   * TODO: actually delete the observation data and return only deleted observation IDs to clients
+   */
+  Archived = 'archive',
+}
+
+/**
+ * TODO: State changes should have a timestamp if we are bothering to track them.
+ */
 export interface ObservationState {
   id: string | PendingEntityId
-  name: 'active' | 'archive'
+  name: ObservationStateName
   userId?: UserId | undefined
   /**
    * @deprecated TODO: confine URLs to the web layer
diff --git a/service/src/routes/observations.js b/service/src/routes/observations.js
index a1d839b12..9091d772a 100644
--- a/service/src/routes/observations.js
+++ b/service/src/routes/observations.js
@@ -5,16 +5,12 @@ module.exports = function (app, security) {
     , archiver = require('archiver')
     , path = require('path')
     , environment = require('../environment/env')
-    , moment = require('moment')
     , access = require('../access')
     , { default: turfCentroid } = require('@turf/centroid')
-    , geometryFormat = require('../format/geoJsonFormat')
     , observationXform = require('../transformers/observation')
     , passport = security.authentication.passport
     , { defaultEventPermissionsService: eventPermissions } = require('../permissions/permissions.events');
 
-  const sortColumnWhitelist = ["lastModified"];
-
   function transformOptions(req) {
     return {
       event: req.event,
@@ -113,84 +109,6 @@ module.exports = function (app, security) {
     });
   }
 
-  function parseQueryParams(req, res, next) {
-    // setup defaults
-    const parameters = {
-      filter: {
-      }
-    };
-
-    const fields = req.query.fields;
-    if (fields) {
-      parameters.fields = JSON.parse(fields);
-    }
-
-    const startDate = req.query.startDate;
-    if (startDate) {
-      parameters.filter.startDate = moment(startDate).utc().toDate();
-    }
-
-    const endDate = req.query.endDate;
-    if (endDate) {
-      parameters.filter.endDate = moment(endDate).utc().toDate();
-    }
-
-    const observationStartDate = req.query.observationStartDate;
-    if (observationStartDate) {
-      parameters.filter.observationStartDate = moment(observationStartDate).utc().toDate();
-    }
-
-    const observationEndDate = req.query.observationEndDate;
-    if (observationEndDate) {
-      parameters.filter.observationEndDate = moment(observationEndDate).utc().toDate();
-    }
-
-    const bbox = req.query.bbox;
-    if (bbox) {
-      parameters.filter.geometries = geometryFormat.parse('bbox', bbox);
-    }
-
-    const geometry = req.query.geometry;
-    if (geometry) {
-      parameters.filter.geometries = geometryFormat.parse('geometry', geometry);
-    }
-
-    const states = req.query.states;
-    if (states) {
-      parameters.filter.states = states.split(',');
-    }
-
-    const sort = req.query.sort;
-    if (sort) {
-      const columns = {};
-      let err = null;
-      sort.split(',').every(function (column) {
-        const sortParams = column.split('+');
-        // Check sort column is in whitelist
-        if (sortColumnWhitelist.indexOf(sortParams[0]) === -1) {
-          err = `Cannot sort on column '${sortParams[0]}'`;
-          return false; // break
-        }
-        // Order can be nothing (ASC by default) or ASC, DESC
-        let direction = 1; // ASC
-        if (sortParams.length > 1 && sortParams[1] === 'DESC') {
-          direction = -1; // DESC
-        }
-        columns[sortParams[0]] = direction;
-      });
-      if (err) {
-        return res.status(400).send(err);
-      }
-      parameters.sort = columns;
-    }
-
-    parameters.populate = req.query.populate === 'true';
-
-    req.parameters = parameters;
-
-    next();
-  }
-
   app.get(
     '/api/events/:eventId/observations/(:observationId).zip',
     passport.authenticate('bearer'),
@@ -239,26 +157,6 @@ module.exports = function (app, security) {
     }
   );
 
-  app.get(
-    '/api/events/:eventId/observations',
-    passport.authenticate('bearer'),
-    validateObservationReadAccess,
-    parseQueryParams,
-    function (req, res, next) {
-      const options = {
-        filter: req.parameters.filter,
-        fields: req.parameters.fields,
-        sort: req.parameters.sort,
-        populate: req.parameters.populate
-      };
-
-      new api.Observation(req.event).getAll(options, function (err, observations) {
-        if (err) return next(err);
-        res.json(observationXform.transform(observations, transformOptions(req)));
-      });
-    }
-  );
-
   app.put(
     '/api/events/:eventId/observations/:observationIdInPath/favorite',
     passport.authenticate('bearer'),
diff --git a/service/test/adapters/observations/adapters.observations.controllers.web.test.ts b/service/test/adapters/observations/adapters.observations.controllers.web.test.ts
index 11e43f0fc..45685686e 100644
--- a/service/test/adapters/observations/adapters.observations.controllers.web.test.ts
+++ b/service/test/adapters/observations/adapters.observations.controllers.web.test.ts
@@ -166,6 +166,45 @@ describe('observations web controller', function () {
     })
   })
 
+  describe('GET /observations', function() {
+
+    describe('query string parsing', function() {
+
+      it('parses bbox as a json string', async function() {
+        // TODO: this format should be deprecated
+        expect.fail('todo')
+      })
+
+      it('parses bbox as csv', async function() {
+        expect.fail('todo')
+      })
+
+      it('parses min last modified date', async function() {
+        expect.fail('todo')
+      })
+
+      it('parses max last modified date', async function() {
+        expect.fail('todo')
+      })
+
+      it('parses min timestamp', async function() {
+        expect.fail('todo')
+      })
+
+      it('parses max timestamp', async function() {
+        expect.fail('todo')
+      })
+
+      it('parses populate flag', async function() {
+        expect.fail('todo')
+      })
+
+      it('parses sort field', async function() {
+        expect.fail('todo')
+      })
+    })
+  })
+
   describe('GET /observations/{observationId}', function() {
 
     it('has tests', async function() {
diff --git a/service/test/app/observations/app.observations.test.ts b/service/test/app/observations/app.observations.test.ts
index 7a3ffe8a4..142fb381b 100644
--- a/service/test/app/observations/app.observations.test.ts
+++ b/service/test/app/observations/app.observations.test.ts
@@ -1545,13 +1545,13 @@ describe('observations use case interactions', function() {
 
   describe('reading observations', function() {
 
-    describe('reading by id', function() {
+    describe('finding one by id', function() {
       it('has tests', async function() {
         expect.fail('todo')
       })
     })
 
-    describe('searching', function() {
+    describe('reading many', function() {
       it('has tests', async function() {
         expect.fail('todo')
       })

From 4aa5b8cd344c10ba46e882b734465b2a20268fa2 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 10 Aug 2024 07:36:16 -0600
Subject: [PATCH 043/183] refactor(service): delete unused observation update
 functions

---
 service/src/api/observation.js    | 169 ------------------------------
 service/src/models/observation.js |  69 ------------
 2 files changed, 238 deletions(-)

diff --git a/service/src/api/observation.js b/service/src/api/observation.js
index d804f1bc3..355db3e4b 100644
--- a/service/src/api/observation.js
+++ b/service/src/api/observation.js
@@ -47,7 +47,6 @@ Observation.prototype.getAll = function(options, callback) {
 /**
  * TODO: this can be deleted when these refs go away
  * * routes/index.js
- * * routes/observations.js
  */
 Observation.prototype.getById = function(observationId, options, callback) {
   if (typeof options === 'function') {
@@ -58,174 +57,6 @@ Observation.prototype.getById = function(observationId, options, callback) {
   ObservationModel.getObservationById(this._event, observationId, options, callback);
 };
 
-Observation.prototype.validate = function(observation) {
-  const errors = {};
-  let message = '';
-
-  if (observation.type !== 'Feature') {
-    errors.type = { error: 'required', message: observation.type ? 'type is required' :  'type must equal "Feature"' };
-    message += observation.type ? '\u2022 type is required\n' : '\u2022 type must equal "Feature"\n'
-  }
-
-  // validate timestamp
-  const properties = observation.properties || {};
-  const timestampError = fieldFactory.createField({
-    type: 'date',
-    required: true,
-    name: 'timestamp',
-    title: 'Date'
-  }, properties).validate();
-  if (timestampError) {
-    errors.timestamp = timestampError;
-    message += `\u2022 ${timestampError.message}\n`;
-  }
-
-  // validate geometry
-  const geometryError = fieldFactory.createField({
-    type: 'geometry',
-    required: true,
-    name: 'geometry',
-    title: 'Location'
-  }, observation).validate();
-  if (geometryError) {
-    errors.geometry = geometryError;
-    message += `\u2022 ${geometryError.message}\n`;
-  }
-
-  const formEntries = properties.forms || [];
-  const formCount = formEntries.reduce((count, form) => {
-    count[form.formId] = (count[form.formId] || 0) + 1;
-    return count;
-  }, {})
-
-  const formDefinitions = {};
-
-  // Validate total number of forms
-  if (this._event.minObservationForms != null && formEntries.length < this._event.minObservationForms) {
-    errors.minObservationForms = new Error("Insufficient number of forms");
-    message += `\u2022 Total number of forms in observation must be at least ${this._event.minObservationForms}\n`;
-  }
-
-  if (this._event.maxObservationForms != null && formEntries.length > this._event.maxObservationForms) {
-    errors.maxObservationForms = new Error("Exceeded maximum number of forms");
-    message += `\u2022 Total number of forms in observation cannot be more than ${this._event.maxObservationForms}\n`;
-  }
-
-  // Validate forms min/max occurrences
-  const formError = {};
-  this._event.forms
-    .filter(form => !form.archived)
-    .forEach(formDefinition => {
-      formDefinitions[formDefinition._id] = formDefinition;
-
-      const count = formCount[formDefinition.id] || 0;
-      if (formDefinition.min && count < formDefinition.min) {
-        formError[formDefinition.id] = {
-          error: 'min',
-          message: `${formDefinition.name} form must be included in observation at least ${formDefinition.min} times`
-        }
-
-        message += `\u2022 ${formDefinition.name} form must be included in observation at least ${formDefinition.min} times\n`;
-      } else if  (formDefinition.max && (count > formDefinition.max)) {
-        formError[formDefinition.id] = {
-          error: 'max',
-          message: `${formDefinition.name} form cannot be included in observation more than ${formDefinition.max} times`
-        }
-
-        message += `\u2022 ${formDefinition.name} form cannot be included in observation more than ${formDefinition.max} times\n`;
-      }
-    });
-
-  // TODO attachment-work, validate attachment restrictions and min/max
-
-  if (Object.keys(formError).length) {
-    errors.form = formError;
-  }
-
-  // Validate form fields
-  const formErrors = [];
-
-  formEntries.forEach(formEntry => {
-    let fieldsMessage = '';
-    const fieldsError = {};
-
-    formDefinitions[formEntry.formId].fields
-      .filter(fieldDefinition => !fieldDefinition.archived)
-      .forEach(fieldDefinition => {
-        const field = fieldFactory.createField(fieldDefinition, formEntry, observation);
-        const fieldError = field.validate();
-
-        if (fieldError) {
-          fieldsError[field.name] = fieldError;
-          fieldsMessage += `    \u2022 ${fieldError.message}\n`;
-        }
-      });
-
-    if (Object.keys(fieldsError).length) {
-      formErrors.push(fieldsError);
-      message += `${formDefinitions[formEntry.formId].name} form is invalid\n`;
-      message += fieldsMessage;
-    }
-  });
-
-  if (formErrors.length) {
-    errors.forms = formErrors
-  }
-
-  if (Object.keys(errors).length) {
-    const err = new Error('Invalid Observation');
-    err.name = 'ValidationError';
-    err.status = 400;
-    err.message = message;
-    err.errors = errors;
-    return err;
-  }
-};
-
-// TODO create is gone, do I need to figure out if this is an observation create?
-Observation.prototype.update = function(observationId, observation, callback) {
-  if (this._user) observation.userId = this._user._id;
-  if (this._deviceId) observation.deviceId = this._deviceId;
-
-  const err = this.validate(observation);
-  if (err) return callback(err);
-
-  ObservationModel.updateObservation(this._event, observationId, observation, (err, updatedObservation) => {
-    if (updatedObservation) {
-      EventEmitter.emit(ObservationEvents.events.update, updatedObservation.toObject({event: this._event}), this._event, this._user);
-
-      // Remove any deleted attachments from file system
-      /*
-      TODO: this might not even work. observation form entries are not stored
-      with field entry keys for attachment fields, so finding attachments based
-      on matching form entry keys with form field names will not produce any
-      results.
-      */
-      const {forms: formEntries = []}  = observation.properties || {};
-      formEntries.forEach(formEntry => {
-        const formDefinition = this._event.forms.find(form => form._id === formEntry.formId);
-        Object.keys(formEntry).forEach(fieldName => {
-          const fieldDefinition = formDefinition.fields.find(field => field.name === fieldName);
-          if (fieldDefinition && fieldDefinition.type === 'attachment') {
-            const attachmentsField = formEntry[fieldName] || [];
-            attachmentsField.filter(attachmentField => attachmentField.action === 'delete').forEach(attachmentField => {
-              const attachment = observation.attachments.find(attachment => attachment._id.toString() === attachmentField.id);
-              if (attachment) {
-                console.info(`deleting attachment ${attachment.id} for field ${fieldName} on observation ${observation.id}`)
-                new Attachment(this._event, observation).delete(attachment._id, err => {
-                  log.warn('Error removing deleted attachment from file system', err);
-                });
-              }
-            });
-          }
-        });
-      });
-    }
-
-    callback(err, updatedObservation);
-  });
-};
-
 Observation.prototype.addFavorite = function(observationId, user, callback) {
   ObservationModel.addFavorite(this._event, observationId, user, callback);
 };
diff --git a/service/src/models/observation.js b/service/src/models/observation.js
index 9a9440eee..9c74545dc 100644
--- a/service/src/models/observation.js
+++ b/service/src/models/observation.js
@@ -137,75 +137,6 @@ exports.getObservationById = function (event, observationId, options, callback)
   observationModel(event).findById(observationId, fields, callback);
 };
 
-exports.updateObservation = function (event, observationId, update, callback) {
-  let addAttachments = [];
-  let removeAttachments = [];
-  const { forms = [] } = update.properties || {};
-  forms.map(observationForm => {
-    observationForm._id = observationForm._id || new mongoose.Types.ObjectId();
-    delete observationForm.id;
-    return observationForm;
-  })
-    .forEach(formEntry => {
-      // TODO: move to app or web layer
-      const formDefinition = event.forms.find(form => form._id === formEntry.formId);
-      Object.keys(formEntry).forEach(fieldName => {
-        const fieldDefinition = formDefinition.fields.find(field => field.name === fieldName);
-        if (fieldDefinition && fieldDefinition.type === 'attachment') {
-          const attachments = formEntry[fieldName] || [];
-          addAttachments = addAttachments.concat(attachments
-            .filter(attachment => attachment.action === 'add')
-            .map(attachment => {
-              return {
-                observationFormId: formEntry._id,
-                fieldName: fieldName,
-                name: attachment.name,
-                contentType: attachment.contentType
-              }
-            }));
-
-          removeAttachments = removeAttachments.concat(attachments
-            .filter(attachment => attachment.action === 'delete')
-            .map(attachment => attachment.id)
-          );
-
-          delete formEntry[fieldName]
-        }
-      });
-    });
-
-  const ObservationModel = observationModel(event);
-  ObservationModel.findById(observationId)
-    .then(observation => {
-      if (!observation) {
-        observation = new ObservationModel({
-          _id: observationId,
-          userId: update.userId,
-          deviceId: update.deviceId,
-          states: [{ name: 'active', userId: update.userId }]
-        });
-      }
-
-      observation.type = update.type;
-      observation.geometry = update.geometry;
-      observation.properties = update.properties;
-      observation.attachments = observation.attachments.concat(addAttachments);
-      observation.attachments = observation.attachments.filter(attachment => {
-        return !removeAttachments.includes(attachment._id.toString())
-      });
-
-      return observation.save();
-    })
-    .then(observation => {
-      return observation
-        .populate([{ path: 'userId', select: 'displayName' }, { path: 'important.userId', select: 'displayName' }]);
-    })
-    .then(observation => {
-      callback(null, observation);
-    })
-    .catch(err => callback(err));
-};
-
 exports.removeObservation = function (event, observationId, callback) {
   observationModel(event).findByIdAndRemove(observationId, callback);
 };

From 955a5c14d3db4a37f7c1dc8f522beaa10eda4a6e Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 12 Aug 2024 13:55:31 -0600
Subject: [PATCH 044/183] refactor(service): implementations for querying
 observations

---
 .../base/adapters.base.db.mongoose.ts         |   2 +-
 .../adapters.observations.db.mongoose.ts      | 104 +++++++++++++++---
 .../observations/app.api.observations.ts      |  24 +---
 .../observations/app.impl.observations.ts     |  17 +++
 .../observations/entities.observations.ts     |  67 ++++++++++-
 .../adapters.observations.db.mongoose.test.ts |  29 +++--
 .../app/observations/app.observations.test.ts |   7 ++
 7 files changed, 203 insertions(+), 47 deletions(-)

diff --git a/service/src/adapters/base/adapters.base.db.mongoose.ts b/service/src/adapters/base/adapters.base.db.mongoose.ts
index c4a3a4424..3401966e4 100644
--- a/service/src/adapters/base/adapters.base.db.mongoose.ts
+++ b/service/src/adapters/base/adapters.base.db.mongoose.ts
@@ -10,7 +10,7 @@ export type WithMongooseDefaultVersionKey = { [MongooseDefaultVersionKey]: numbe
 /**
  * Map Mongoose `Document` instances to plain entity objects.
  */
-export type DocumentMapping<DocType extends mongoose.AnyObject, E extends object> = (doc: DocType | mongoose.HydratedDocument<DocType>) => E
+export type DocumentMapping<DocType extends mongoose.AnyObject, E extends object> = (doc: DocType | mongoose.HydratedDocument<DocType> | mongoose.LeanDocument<DocType>) => E
 /**
  * Map entities to objects suitable to create Mongoose `Model` `Document` instances, as
  * in `new mongoose.Model(stub)`.
diff --git a/service/src/adapters/observations/adapters.observations.db.mongoose.ts b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
index 5a6324a9e..de9d48274 100644
--- a/service/src/adapters/observations/adapters.observations.db.mongoose.ts
+++ b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
@@ -1,5 +1,5 @@
 import { MageEvent, MageEventAttrs, MageEventId } from '../../entities/events/entities.events'
-import { Attachment, AttachmentId, AttachmentNotFoundError, AttachmentPatchAttrs, copyObservationAttrs, EventScopedObservationRepository, FormEntry, FormEntryId, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationState, patchAttachment, Thumbnail } from '../../entities/observations/entities.observations'
+import { Attachment, AttachmentId, AttachmentNotFoundError, AttachmentPatchAttrs, copyObservationAttrs, EventScopedObservationRepository, UsersExpandedObservationAttrs, FindObservationsSpec, FormEntry, FormEntryId, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationState, patchAttachment, Thumbnail, UserExpandedObservationImportantFlag } from '../../entities/observations/entities.observations'
 import { BaseMongooseRepository, DocumentMapping, pageQuery, WithMongooseDefaultVersionKey } from '../base/adapters.base.db.mongoose'
 import mongoose from 'mongoose'
 import { MageEventDocument } from '../../models/event'
@@ -293,8 +293,7 @@ export class MongooseObservationRepository extends BaseMongooseRepository<Observ
         console.warn(`attempted to modify create timestamp on observation ${beforeDoc.id} from ${beforeDoc.createdAt} to ${docSeed.createdAt}`)
         docSeed.createdAt = new Date(beforeDoc.createdAt)
       }
-      //TODO remove any, was as legacy.ObservationDocument
-      beforeDoc = beforeDoc.set(docSeed) as any
+      beforeDoc = beforeDoc.set(docSeed)
     }
     else {
       const idVerified = await this.idModel.findById(dbId)
@@ -329,6 +328,63 @@ export class MongooseObservationRepository extends BaseMongooseRepository<Observ
     return latest ? this.entityForDocument(latest) : null
   }
 
+  async findSome<
+    FindSpec extends FindObservationsSpec = FindObservationsSpec,
+    T = FindSpec extends { populateUserNames: true } ? UsersExpandedObservationAttrs : ObservationAttrs
+  >(findSpec: FindSpec, mapping?: (x: ObservationAttrs | UsersExpandedObservationAttrs) => T): Promise<PageOf<T>> {
+    const { where, orderBy, paging: specPaging } = findSpec
+    const dbFilter = {} as any
+    if (where.lastModifiedAfter) {
+      dbFilter.lastModified = { $gte: where.lastModifiedAfter }
+    }
+    if (where.lastModifiedBefore) {
+      dbFilter.lastModified = { ...dbFilter.lastModified, $lt: where.lastModifiedBefore }
+    }
+    if (where.timestampAfter) {
+      dbFilter['properties.timestamp'] = { $gte: where.timestampAfter }
+    }
+    if (where.timestampBefore) {
+      dbFilter['properties.timestamp'] = { ...dbFilter['properties.timestamp'], $lt: where.timestampBefore }
+    }
+    if (where.stateIsAnyOf) {
+      dbFilter['states.0.name'] = { $in: where.stateIsAnyOf }
+    }
+    if (typeof where.isFlaggedImportant === 'boolean') {
+      dbFilter.important = { $exists: where.isFlaggedImportant }
+    }
+    if (Array.isArray(where.isFavoriteOfUsers) && where.isFavoriteOfUsers.length > 0) {
+      dbFilter.favoriteUserIds = { $in: where.isFavoriteOfUsers }
+    }
+    const dbSort = {} as any
+    const order = typeof orderBy?.order === 'number' ? orderBy.order : 1
+    if (orderBy?.field === 'lastModified') {
+      dbSort.lastModified = order
+    }
+    else if (orderBy?.field === 'timestamp') {
+      dbSort['properties.timestamp'] = order
+    }
+    // add _id to sort for consistent ordering
+    dbSort._id = orderBy?.order || -1
+    const populatedUserNullSafetyTransform = (user: object | null, _id: mongoose.Types.ObjectId): object => user ? user : { _id }
+    const options: mongoose.QueryOptions<ObservationDocument> = dbSort ? { sort: dbSort } : {}
+    const paging: PagingParameters = specPaging || { includeTotalCount: false, pageSize: Number.MAX_SAFE_INTEGER, pageIndex: 0 }
+    const counted = await pageQuery(
+      this.model.find(dbFilter, null, options)
+        .lean(true)
+        .populate([
+          // TODO: something smarter than a mongoose join would be good here
+          // maybe just store the display name on the observation document
+          { path: 'userId', select: 'displayName', transform: populatedUserNullSafetyTransform },
+          { path: 'important.userId', select: 'displayName', transform: populatedUserNullSafetyTransform }
+        ]),
+      paging
+    )
+    const mapResult = typeof mapping === 'function' ? mapping : ((x: any): T => x as T)
+    const items = await counted.query
+    const mappedItems = items.map(doc => mapResult(this.entityForDocument(doc)))
+    return pageOf(mappedItems, paging, counted.totalCount)
+  }
+
   async findLastModifiedAfter(timestamp: number, paging: PagingParameters): Promise<PageOf<ObservationAttrs>> {
     const match = { lastModified: {$gte: new Date(timestamp)} }
     const counted = await pageQuery(this.model.find(match), paging)
@@ -336,7 +392,6 @@ export class MongooseObservationRepository extends BaseMongooseRepository<Observ
     for await (const doc of counted.query.cursor()) {
       observations.push(this.entityForDocument(doc))
     }
-
     return pageOf(observations, paging, counted.totalCount)
   }
 
@@ -365,36 +420,47 @@ export class MongooseObservationRepository extends BaseMongooseRepository<Observ
   }
 }
 
-export function docToEntity(doc: ObservationDocument, eventId: MageEventId): ObservationAttrs {
-  return createDocumentMapping(eventId)(doc)
+type PopulatedUserDocument = { _id: mongoose.Types.ObjectId, displayName: string }
+type ObservationDocumentPopulated = Omit<ObservationDocument, 'userId' | 'important'> & {
+  userId: PopulatedUserDocument
+  important: Omit<ObservationDocumentImportantFlag, 'userId'> & { userId: PopulatedUserDocument }
 }
 
-function createDocumentMapping(eventId: MageEventId): DocumentMapping<ObservationDocument, ObservationAttrs> {
+function createDocumentMapping(eventId: MageEventId): DocumentMapping<ObservationDocument | ObservationDocumentPopulated, ObservationAttrs> {
   return doc => {
-    const attrs: ObservationAttrs = {
+    const attrs: UsersExpandedObservationAttrs = {
       id: doc._id.toHexString(),
       eventId,
       createdAt: doc.createdAt,
       lastModified: doc.lastModified,
       type: doc.type,
       geometry: doc.geometry,
-      bbox: doc.bbox,
+      bbox: doc.bbox as GeoJSON.BBox,
       states: doc.states.map(stateAttrsForDoc),
       properties: {
         ...doc.properties,
         forms: doc.properties.forms.map(formEntryForDoc)
       },
+      important: importantFlagAttrsForDoc(doc) as any,
       attachments: doc.attachments.map(attachmentAttrsForDoc),
-      userId: doc.userId?.toHexString(),
       deviceId: doc.deviceId?.toHexString(),
-      important: importantFlagAttrsForDoc(doc),
       favoriteUserIds: doc.favoriteUserIds?.map(x => x.toHexString()),
     }
+    if (doc.userId instanceof mongoose.Types.ObjectId) {
+      attrs.userId = doc.userId.toHexString()
+    }
+    else if (doc.userId?._id) {
+      attrs.userId = doc.userId._id.toHexString()
+      attrs.user = {
+        id: attrs.userId,
+        displayName: doc.userId.displayName
+      }
+    }
     return attrs
   }
 }
 
-function importantFlagAttrsForDoc(doc: ObservationDocument): ObservationImportantFlag | undefined {
+function importantFlagAttrsForDoc(doc: ObservationDocument | ObservationDocumentPopulated | mongoose.LeanDocument<ObservationDocument | ObservationDocumentPopulated>): ObservationImportantFlag | UserExpandedObservationImportantFlag | undefined {
   /*
   because the observation schema defines `important` as a nested document
   instead of a subdocument schema, a mongoose observation document instance
@@ -405,11 +471,21 @@ function importantFlagAttrsForDoc(doc: ObservationDocument): ObservationImportan
   */
   const docImportant = doc.important
   if (docImportant?.userId || docImportant?.timestamp || docImportant?.description) {
-    return {
-      userId: docImportant.userId?.toHexString(),
+    const important: UserExpandedObservationImportantFlag = {
       timestamp: docImportant.timestamp,
       description: docImportant.description
     }
+    if (docImportant.userId instanceof mongoose.Types.ObjectId) {
+      important.userId = docImportant.userId.toHexString()
+    }
+    else if (docImportant.userId?._id instanceof mongoose.Types.ObjectId) {
+      important.userId = docImportant.userId._id.toHexString()
+      important.user = {
+        id: docImportant.userId._id.toHexString(),
+        displayName: docImportant.userId.displayName
+      }
+    }
+    return important
   }
   return void(0)
 }
diff --git a/service/src/app.api/observations/app.api.observations.ts b/service/src/app.api/observations/app.api.observations.ts
index bf6871627..cd61c6cdb 100644
--- a/service/src/app.api/observations/app.api.observations.ts
+++ b/service/src/app.api/observations/app.api.observations.ts
@@ -1,8 +1,9 @@
 import { EntityNotFoundError, InfrastructureError, InvalidInputError, PermissionDeniedError } from '../app.api.errors'
 import { AppRequest, AppRequestContext, AppResponse } from '../app.api.global'
-import { Attachment, AttachmentId, copyObservationAttrs, EventScopedObservationRepository, FormEntry, FormFieldEntry, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationState, StagedAttachmentContentRef, Thumbnail, thumbnailIndexForTargetDimension } from '../../entities/observations/entities.observations'
+import { Attachment, AttachmentId, copyObservationAttrs, EventScopedObservationRepository, FindObservationsSpec, FormEntry, FormFieldEntry, Observation, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationState, StagedAttachmentContentRef, Thumbnail, thumbnailIndexForTargetDimension } from '../../entities/observations/entities.observations'
 import { MageEvent } from '../../entities/events/entities.events'
 import { User, UserId } from '../../entities/users/entities.users'
+import { PageOf } from '../../entities/entities.global'
 
 
 
@@ -44,26 +45,11 @@ export interface ReadObservation {
 }
 
 export interface ReadObservationsRequest extends ObservationRequest {
-  filter: {
-    lastModifiedAfter?: Date | undefined,
-    lastModifiedBefore?: Date | undefined,
-    timestampAfter?: Date | undefined,
-    timestampBefore?: Date | undefined,
-    bbox?: GeoJSON.BBox,
-    states?: ObservationState['name'][] | undefined,
-  },
-  sort?: {
-    field: string,
-    order?: 1 | -1,
-  },
-  /**
-   * If `true`, populate the user names for the observation {@link ObservationAttrs.userId creator} and
-   * {@link ObservationImportantFlag.userId important flag}.
-   */
-  populate?: boolean | undefined
+  findSpec: FindObservationsSpec,
+  mapping?: <T>(x: ExoObservation) => T
 }
 export interface ReadObservations {
-  (req: ReadObservationsRequest): Promise<AppResponse<ExoObservation[], PermissionDeniedError | InvalidInputError>>
+  (req: ReadObservationsRequest): Promise<AppResponse<PageOf<ExoObservation[]>, PermissionDeniedError | InvalidInputError | InfrastructureError>>
 }
 
 export interface StoreAttachmentContent {
diff --git a/service/src/app.impl/observations/app.impl.observations.ts b/service/src/app.impl/observations/app.impl.observations.ts
index 6135f3dfb..1f44eeb3f 100644
--- a/service/src/app.impl/observations/app.impl.observations.ts
+++ b/service/src/app.impl/observations/app.impl.observations.ts
@@ -52,6 +52,23 @@ export function SaveObservation(permissionService: api.ObservationPermissionServ
   }
 }
 
+export function ReadObservations(permissionService: api.ObservationPermissionService): api.ReadObservations {
+  return async function readObservations(req: api.ReadObservationsRequest): ReturnType<api.ReadObservations> {
+    const denied = await permissionService.ensureReadObservationPermission(req.context)
+    if (denied) {
+      return AppResponse.error(denied)
+    }
+    const mapping = (x: ObservationAttrs): any => (typeof req.mapping === 'function' ? req.mapping(api.exoObservationFor(x)) : api.exoObservationFor(x))
+    try {
+      const results = await req.context.observationRepository.findSome(req.findSpec, mapping)
+      return AppResponse.success(results)
+    }
+    catch (err) {
+      return AppResponse.error(infrastructureError(err instanceof Error ? err : String(err)))
+    }
+  }
+}
+
 export function ReadObservation(permissionService: api.ObservationPermissionService): api.ReadObservation {
   return async function readObservation(req: api.ReadObservationRequest): ReturnType<api.ReadObservation> {
     const denied = await permissionService.ensureReadObservationPermission(req.context)
diff --git a/service/src/entities/observations/entities.observations.ts b/service/src/entities/observations/entities.observations.ts
index 00d5c5f86..1091cfc59 100644
--- a/service/src/entities/observations/entities.observations.ts
+++ b/service/src/entities/observations/entities.observations.ts
@@ -1,4 +1,4 @@
-import { UserId } from '../users/entities.users'
+import { User, UserId } from '../users/entities.users'
 import { BBox, Feature, Geometry } from 'geojson'
 import { MageEvent, MageEventAttrs, MageEventId } from '../events/entities.events'
 import { PageOf, PagingParameters, PendingEntityId } from '../entities.global'
@@ -630,10 +630,10 @@ export function validationResultMessage(result: ObservationValidationResult): st
   if (totalFormCountError) {
     errList.push(`${bulletPoint} ${totalFormCountError.message()}`)
   }
-  for (const [ formId, err ] of formCountErrors) {
+  for (const [ , err ] of formCountErrors) {
     errList.push(`${bulletPoint} ${err.message()}`)
   }
-  for (const [ formEntryId, formEntryErr ] of formEntryErrors) {
+  for (const [ , formEntryErr ] of formEntryErrors) {
     errList.push(`${bulletPoint} Form entry ${formEntryErr.formEntryPosition + 1} (${formEntryErr.formName}) is invalid.`)
     for (const fieldErr of formEntryErr.fieldErrors.values()) {
       errList.push(`  ${bulletPoint} ${fieldErr.message}`)
@@ -857,7 +857,13 @@ export interface EventScopedObservationRepository {
    * @returns an `Observation` object or `null`
    */
   findLatest(): Promise<ObservationAttrs | null>
+  /**
+   * @deprecated TODO: `findSome()` makes this obsolete.  One of the plugins might use this one.
+   */
   findLastModifiedAfter(timestamp: number, paging: PagingParameters): Promise<PageOf<ObservationAttrs>>
+  findSome<T = ObservationAttrs>(findSpec: FindObservationsSpecUnpopulated, mapping?: (o: ObservationAttrs) => T): Promise<PageOf<T>>
+  findSome<T = UsersExpandedObservationAttrs>(findSpec: FindObservationsSpecPopulated, mapping?: (o: UsersExpandedObservationAttrs) => T): Promise<PageOf<T>>
+  findSome<T = ObservationAttrs>(findSpec: FindObservationsSpec, mapping?: (o: ObservationAttrs | UsersExpandedObservationAttrs) => T): Promise<PageOf<T>>
   /**
    * Update the specified attachment with the given attributes.  This
    * persistence function exists alongside the {@link save} method to prevent
@@ -891,8 +897,59 @@ export interface EventScopedObservationRepository {
   nextAttachmentIds(count?: number): Promise<AttachmentId[]>
 }
 
-export class ObservationRepositoryError extends Error {
+export type FindObservationsSortField = 'lastModified' | 'timestamp'
 
+export interface FindObservationsSort {
+  /**
+   * The default sort field is `lastModified`.
+   */
+  field: FindObservationsSortField
+  /**
+   * `1` indicates ascending, `-1` indicates descending.  Ascending is the default order.
+   */
+  order?: 1 | -1
+}
+export interface FindObservationsSpec {
+  where: {
+    lastModifiedAfter?: Date
+    lastModifiedBefore?: Date
+    timestampAfter?: Date
+    timestampBefore?: Date
+    /**
+     * A series of lon/lat coordinates in the order [ west, south, east, north ] as in
+     * https://datatracker.ietf.org/doc/html/rfc7946#section-5.
+     */
+    locationIntersects?: [ number, number, number, number ]
+    stateIsAnyOf?: ObservationStateName[]
+    /**
+     * Specifying a boolean value finds only observations conforming to the value, whereas omitting or specifiying
+     * `null` causes the query to disregard the important flag.
+     */
+    isFlaggedImportant?: boolean | null,
+    isFavoriteOfUsers?: UserId[],
+  }
+  /**
+   * If `true`, populate the display names from related user documents on the observation creator and important flag.
+   * This results in adding `user: { id: string, displayName: string }` entries on the observation document and
+   * important sub-document.
+   */
+  populateUserNames?: boolean
+  orderBy?: FindObservationsSort
+  paging?: PagingParameters
+}
+type FindObservationsSpecPopulated = FindObservationsSpec & { populateUserNames: true }
+type FindObservationsSpecUnpopulated = Omit<FindObservationsSpec, 'populateUserNames'> | (FindObservationsSpec & { populateUserNames: false })
+
+export type ObservationUserExpanded = Pick<User, 'id' | 'displayName'>
+
+export type UsersExpandedObservationAttrs = ObservationAttrs & {
+  user?: ObservationUserExpanded
+  important?: UserExpandedObservationImportantFlag
+}
+
+export type UserExpandedObservationImportantFlag = ObservationImportantFlag & { user?: ObservationUserExpanded }
+
+export class ObservationRepositoryError extends Error {
   constructor(readonly code: ObservationRepositoryErrorCode, message?: string) {
     super(message)
   }
@@ -1186,7 +1243,7 @@ const FieldTypeValidationRules: { [type in FormFieldType]: FormFieldValidationRu
   [FormFieldType.Email]: validateRequiredThen(context => fields.email.EmailFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))),
   [FormFieldType.Geometry]: validateRequiredThen(context => fields.geometry.GeometryFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))),
   // TODO: no validation at all? legacy validation code did nothing for hidden fields
-  [FormFieldType.Hidden]: context => null,
+  [FormFieldType.Hidden]: () => null,
   [FormFieldType.MultiSelectDropdown]: validateRequiredThen(context => fields.multiselect.MultiSelectFormFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))),
   [FormFieldType.Numeric]: validateRequiredThen(context => fields.numeric.NumericFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))),
   [FormFieldType.Password]: validateRequiredThen(context => fields.text.TextFieldValidation(context.field, context.fieldEntry, FormFieldValidationResult(context))),
diff --git a/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts b/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts
index 091ccf479..bf57d8e1b 100644
--- a/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts
+++ b/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts
@@ -3,13 +3,13 @@ import { expect } from 'chai'
 import mongoose from 'mongoose'
 import _ from 'lodash'
 import { MongooseMageEventRepository } from '../../../lib/adapters/events/adapters.events.db.mongoose'
-import { MongooseObservationRepository, ObservationModel } from '../../../lib/adapters/observations/adapters.observations.db.mongoose'
+import { MongooseObservationRepository, ObservationDocument, ObservationModel } from '../../../lib/adapters/observations/adapters.observations.db.mongoose'
 import * as legacy from '../../../lib/models/observation'
 import * as legacyEvent from '../../../lib/models/event'
 import { MageEventDocument } from '../../../src/models/event'
 
 import { MageEvent, MageEventAttrs, MageEventCreateAttrs, MageEventId } from '../../../lib/entities/events/entities.events'
-import { ObservationAttrs, ObservationId, Observation, ObservationRepositoryError, ObservationRepositoryErrorCode, copyObservationAttrs, AttachmentContentPatchAttrs, copyAttachmentAttrs, AttachmentNotFoundError, AttachmentPatchAttrs, removeAttachment, validationResultMessage, ObservationDomainEventType, ObservationEmitted, PendingObservationDomainEvent, AttachmentsRemovedDomainEvent } from '../../../lib/entities/observations/entities.observations'
+import { ObservationAttrs, ObservationId, Observation, ObservationRepositoryError, ObservationRepositoryErrorCode, copyObservationAttrs, AttachmentContentPatchAttrs, copyAttachmentAttrs, AttachmentNotFoundError, AttachmentPatchAttrs, removeAttachment, validationResultMessage, ObservationDomainEventType, ObservationEmitted, PendingObservationDomainEvent, AttachmentsRemovedDomainEvent, ObservationStateName, EventScopedObservationRepository } from '../../../lib/entities/observations/entities.observations'
 import { AttachmentPresentationType, FormFieldType, Form, AttachmentMediaTypes } from '../../../lib/entities/events/entities.events.forms'
 import util from 'util'
 import { PendingEntityId } from '../../../lib/entities/entities.global'
@@ -157,6 +157,19 @@ describe('mongoose observation repository', function() {
     })
   })
 
+  describe('reading observations', function() {
+
+    describe('finding some', function() {
+
+      it('has tests', async function() {
+
+        const some = await repo.findSome({ where: {} })
+
+        expect.fail('todo')
+      })
+    })
+  })
+
   describe('saving observations', function() {
 
     describe('new observations', function() {
@@ -241,12 +254,12 @@ describe('mongoose observation repository', function() {
         attrs.states = [
           {
             id: (new mongoose.Types.ObjectId()).toHexString(),
-            name: 'active',
+            name: ObservationStateName.Active,
             userId: (new mongoose.Types.ObjectId()).toHexString()
           },
           {
             id: (new mongoose.Types.ObjectId()).toHexString(),
-            name: 'archive',
+            name: ObservationStateName.Archived,
             userId: undefined
           }
         ]
@@ -292,7 +305,7 @@ describe('mongoose observation repository', function() {
           }
         ]
         origAttrs.states = [
-          { id: (new mongoose.Types.ObjectId()).toHexString(), name: 'active', userId: (new mongoose.Types.ObjectId()).toHexString() }
+          { id: (new mongoose.Types.ObjectId()).toHexString(), name: ObservationStateName.Active, userId: (new mongoose.Types.ObjectId()).toHexString() }
         ]
         origAttrs.properties.forms = [
           {
@@ -325,7 +338,7 @@ describe('mongoose observation repository', function() {
           coordinates: [ 12, 34 ]
         }
         putAttrs.states = [
-          { name: 'archive', id: PendingEntityId }
+          { name: ObservationStateName.Archived, id: PendingEntityId }
         ]
         putAttrs.properties.forms = [
           {
@@ -411,7 +424,7 @@ describe('mongoose observation repository', function() {
       state1Stub.states = [
         {
           id: PendingEntityId,
-          name: 'archive',
+          name: ObservationStateName.Archived,
           userId: (new mongoose.Types.ObjectId()).toHexString()
         }
       ]
@@ -422,7 +435,7 @@ describe('mongoose observation repository', function() {
       state2Stub.states = [
         {
           id: PendingEntityId,
-          name: 'active',
+          name: ObservationStateName.Active,
           userId: (new mongoose.Types.ObjectId()).toHexString()
         },
         state1Saved.states[0],
diff --git a/service/test/app/observations/app.observations.test.ts b/service/test/app/observations/app.observations.test.ts
index 142fb381b..2da37bd29 100644
--- a/service/test/app/observations/app.observations.test.ts
+++ b/service/test/app/observations/app.observations.test.ts
@@ -1553,6 +1553,13 @@ describe('observations use case interactions', function() {
 
     describe('reading many', function() {
       it('has tests', async function() {
+
+        const req: api.ReadObservationsRequest = {
+          context,
+          findSpec: {
+            where: {}
+          }
+        }
         expect.fail('todo')
       })
     })

From 52b3a004d1e8230c9b8dd23c17fec1f8111a5396 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 12 Aug 2024 16:36:25 -0600
Subject: [PATCH 045/183] refactor(service): add bbox support to observation
 query; change favorite user query to single user id instead of array to match
 original api; clean up types

---
 .../observations/adapters.observations.db.mongoose.ts  |  7 +++++--
 .../src/entities/observations/entities.observations.ts |  4 ++--
 .../adapters.observations.db.mongoose.test.ts          |  6 +++---
 service/test/app/observations/app.observations.test.ts | 10 +++++-----
 4 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/service/src/adapters/observations/adapters.observations.db.mongoose.ts b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
index de9d48274..53c2ecd28 100644
--- a/service/src/adapters/observations/adapters.observations.db.mongoose.ts
+++ b/service/src/adapters/observations/adapters.observations.db.mongoose.ts
@@ -349,11 +349,14 @@ export class MongooseObservationRepository extends BaseMongooseRepository<Observ
     if (where.stateIsAnyOf) {
       dbFilter['states.0.name'] = { $in: where.stateIsAnyOf }
     }
+    if (Array.isArray(where.geometryIntersects)) {
+      dbFilter.$geoIntersects = { $geometry: where.geometryIntersects }
+    }
     if (typeof where.isFlaggedImportant === 'boolean') {
       dbFilter.important = { $exists: where.isFlaggedImportant }
     }
-    if (Array.isArray(where.isFavoriteOfUsers) && where.isFavoriteOfUsers.length > 0) {
-      dbFilter.favoriteUserIds = { $in: where.isFavoriteOfUsers }
+    if (where.isFavoriteOfUser) {
+      dbFilter.favoriteUserIds = where.isFavoriteOfUser
     }
     const dbSort = {} as any
     const order = typeof orderBy?.order === 'number' ? orderBy.order : 1
diff --git a/service/src/entities/observations/entities.observations.ts b/service/src/entities/observations/entities.observations.ts
index 1091cfc59..835bbac76 100644
--- a/service/src/entities/observations/entities.observations.ts
+++ b/service/src/entities/observations/entities.observations.ts
@@ -919,14 +919,14 @@ export interface FindObservationsSpec {
      * A series of lon/lat coordinates in the order [ west, south, east, north ] as in
      * https://datatracker.ietf.org/doc/html/rfc7946#section-5.
      */
-    locationIntersects?: [ number, number, number, number ]
+    geometryIntersects?: [ number, number, number, number ]
     stateIsAnyOf?: ObservationStateName[]
     /**
      * Specifying a boolean value finds only observations conforming to the value, whereas omitting or specifiying
      * `null` causes the query to disregard the important flag.
      */
     isFlaggedImportant?: boolean | null,
-    isFavoriteOfUsers?: UserId[],
+    isFavoriteOfUser?: UserId,
   }
   /**
    * If `true`, populate the display names from related user documents on the observation creator and important flag.
diff --git a/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts b/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts
index bf57d8e1b..1b5387e96 100644
--- a/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts
+++ b/service/test/adapters/observations/adapters.observations.db.mongoose.test.ts
@@ -2,7 +2,7 @@ import { describe, it } from 'mocha'
 import { expect } from 'chai'
 import mongoose from 'mongoose'
 import _ from 'lodash'
-import { MongooseMageEventRepository } from '../../../lib/adapters/events/adapters.events.db.mongoose'
+import { MageEventModel, MongooseMageEventRepository } from '../../../lib/adapters/events/adapters.events.db.mongoose'
 import { MongooseObservationRepository, ObservationDocument, ObservationModel } from '../../../lib/adapters/observations/adapters.observations.db.mongoose'
 import * as legacy from '../../../lib/models/observation'
 import * as legacyEvent from '../../../lib/models/event'
@@ -56,7 +56,7 @@ describe('mongoose observation repository', function() {
 
   beforeEach('initialize model', async function() {
     //TODO remove cast to any, was mongoose.Model<MageEventDocument>
-    const MageEventModel = legacyEvent.Model as any
+    const MageEventModel = legacyEvent.Model
     const eventRepo = new MongooseMageEventRepository(MageEventModel)
     createEvent = (attrs: Partial<MageEventAttrs>): Promise<MageEventDocument> => {
       return new Promise<MageEventDocument>((resolve, reject) => {
@@ -122,7 +122,7 @@ describe('mongoose observation repository', function() {
       userFields: []
     })
     domainEvents = Substitute.for<EventEmitter>()
-    model = legacy.observationModel(eventDoc)
+    model = MageEventModel.
     repo = new MongooseObservationRepository(eventDoc, eventRepo.findById.bind(eventRepo), domainEvents)
     event = new MageEvent(eventRepo.entityForDocument(eventDoc))
 
diff --git a/service/test/app/observations/app.observations.test.ts b/service/test/app/observations/app.observations.test.ts
index 2da37bd29..3a638de78 100644
--- a/service/test/app/observations/app.observations.test.ts
+++ b/service/test/app/observations/app.observations.test.ts
@@ -4,7 +4,7 @@ import uniqid from 'uniqid'
 import * as api from '../../../lib/app.api/observations/app.api.observations'
 import { AllocateObservationId, registerDeleteRemovedAttachmentsHandler, ReadAttachmentContent, SaveObservation, StoreAttachmentContent } from '../../../lib/app.impl/observations/app.impl.observations'
 import { copyMageEventAttrs, MageEvent } from '../../../lib/entities/events/entities.events'
-import { addAttachment, Attachment, AttachmentContentPatchAttrs, AttachmentCreateAttrs, AttachmentsRemovedDomainEvent, AttachmentStore, AttachmentStoreError, AttachmentStoreErrorCode, copyAttachmentAttrs, copyObservationAttrs, copyObservationStateAttrs, EventScopedObservationRepository, Observation, ObservationAttrs, ObservationDomainEventType, ObservationEmitted, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationState, patchAttachment, putAttachmentThumbnailForMinDimension, removeAttachment, removeFormEntry, StagedAttachmentContentRef } from '../../../lib/entities/observations/entities.observations'
+import { addAttachment, Attachment, AttachmentContentPatchAttrs, AttachmentCreateAttrs, AttachmentsRemovedDomainEvent, AttachmentStore, AttachmentStoreError, AttachmentStoreErrorCode, copyAttachmentAttrs, copyObservationAttrs, copyObservationStateAttrs, EventScopedObservationRepository, Observation, ObservationAttrs, ObservationDomainEventType, ObservationEmitted, ObservationRepositoryError, ObservationRepositoryErrorCode, ObservationState, ObservationStateName, patchAttachment, putAttachmentThumbnailForMinDimension, removeAttachment, removeFormEntry, StagedAttachmentContentRef } from '../../../lib/entities/observations/entities.observations'
 import { permissionDenied, MageError, ErrPermissionDenied, ErrEntityNotFound, EntityNotFoundError, InvalidInputError, ErrInvalidInput, PermissionDeniedError, InfrastructureError, ErrInfrastructure } from '../../../lib/app.api/app.api.errors'
 import { FormFieldType } from '../../../lib/entities/events/entities.events.forms'
 import _ from 'lodash'
@@ -290,8 +290,8 @@ describe('observations use case interactions', function() {
       expect(exo.state).to.be.undefined
 
       const states: ObservationState[] = [
-        { id: uniqid(), name: 'archive', userId: uniqid() },
-        { id: uniqid(), name: 'active', userId: uniqid() }
+        { id: uniqid(), name: ObservationStateName.Archived, userId: uniqid() },
+        { id: uniqid(), name: ObservationStateName.Active, userId: uniqid() }
       ]
       from.states = states.map(copyObservationStateAttrs)
       exo = api.exoObservationFor(from)
@@ -1417,8 +1417,8 @@ describe('observations use case interactions', function() {
           ...copyObservationAttrs(obsBefore),
           id: uniqid(),
           states: [
-            { id: uniqid(), name: 'active', userId: uniqid() },
-            { id: uniqid(), name: 'archive', userId: uniqid() }
+            { id: uniqid(), name: ObservationStateName.Active, userId: uniqid() },
+            { id: uniqid(), name: ObservationStateName.Archived, userId: uniqid() }
           ]
         }, mageEvent) as Observation
         const obsAfter = Observation.assignTo(obsBefore, {

From 99158905eed1e3d0b02154104193574955e7e0de Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 15 Aug 2024 10:39:29 -0600
Subject: [PATCH 046/183] refactor(service): devices typescript/di: create
 device entity module

---
 .../src/entities/devices/entities.devices.ts  | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 service/src/entities/devices/entities.devices.ts

diff --git a/service/src/entities/devices/entities.devices.ts b/service/src/entities/devices/entities.devices.ts
new file mode 100644
index 000000000..de5e7f11d
--- /dev/null
+++ b/service/src/entities/devices/entities.devices.ts
@@ -0,0 +1,35 @@
+import { PagingParameters } from '../entities.global'
+import { User, UserId } from '../users/entities.users'
+
+export type DeviceId = string
+export type DeviceUid = string
+
+export interface Device {
+  id: DeviceId
+  /**
+   * The UID originates from the client, whereas the server generates ID.
+   */
+  uid: DeviceUid
+  description?: string | undefined
+  registered: boolean
+  userId: UserId
+  userAgent?: string | undefined
+  appVersion?: string | undefined
+}
+
+export type DeviceExpanded = Device & { user: User | null }
+
+export interface FindDevicesSpec {
+  paging: PagingParameters
+}
+
+export interface DeviceRepository {
+  create(deviceAttrs: Omit<Device, 'id'>): Promise<Device>
+  update(deviceAttrs: Partial<Device>): Promise<Device | null>
+  removeById(id: DeviceId): Promise<Device | null>
+  removeByUid(uid: DeviceUid): Promise<Device | null>
+  findById(id: DeviceId): Promise<null | DeviceExpanded>
+  findByUid(uid: DeviceUid): Promise<null | DeviceExpanded>
+  findSome(findSpec: FindDevicesSpec): Promise<Device[]>
+  countSome(findSpec: FindDevicesSpec): Promise<number>
+}
\ No newline at end of file

From f70cad27a71b2076e6077f9be06a7a5d6cd6b4db Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 15 Aug 2024 16:17:24 -0600
Subject: [PATCH 047/183] refactor(service): devices typescript/di: create
 device mongoose repository

---
 .../devices/adapters.devices.db.mongoose.ts   | 100 ++++++++++++++++++
 .../src/entities/devices/entities.devices.ts  |  36 +++++--
 2 files changed, 129 insertions(+), 7 deletions(-)
 create mode 100644 service/src/adapters/devices/adapters.devices.db.mongoose.ts

diff --git a/service/src/adapters/devices/adapters.devices.db.mongoose.ts b/service/src/adapters/devices/adapters.devices.db.mongoose.ts
new file mode 100644
index 000000000..8d6f5731f
--- /dev/null
+++ b/service/src/adapters/devices/adapters.devices.db.mongoose.ts
@@ -0,0 +1,100 @@
+import _ from 'lodash'
+import mongoose from 'mongoose'
+import { Device, DeviceExpanded, DeviceRepository, DeviceUid, FindDevicesSpec } from '../../entities/devices/entities.devices'
+import { pageOf, PageOf } from '../../entities/entities.global'
+import { BaseMongooseRepository, DocumentMapping, pageQuery } from '../base/adapters.base.db.mongoose'
+import { UserDocument } from '../users/adapters.users.db.mongoose'
+
+const Schema = mongoose.Schema;
+
+export type DeviceDocument = Omit<Device, 'id' | 'userId'> & {
+  _id: mongoose.Types.ObjectId
+  userId: mongoose.Types.ObjectId | null
+}
+export type DeviceDocumentPopulated = Omit<DeviceDocument, 'userId'> & {
+  userId: { _id: UserDocument['_id'], displayName?: UserDocument['displayName'] | null }
+}
+export type DeviceModel = mongoose.Model<DeviceDocument>
+
+// TODO cascade delete from userId when user is deleted.
+export const DeviceSchema = new Schema<DeviceDocument, DeviceModel>(
+  {
+    uid: { type: String, required: true, unique: true, lowercase: true },
+    description: { type: String, required: false },
+    registered: { type: Boolean, required: true, default: false },
+    userId: { type: Schema.Types.ObjectId, ref: 'User' },
+    userAgent: { type: String, required: false },
+    appVersion: { type: String, required: false }
+  },
+  {
+    versionKey: false
+  }
+)
+
+export const docToEntity: DocumentMapping<DeviceDocument | DeviceDocumentPopulated, Device | DeviceExpanded> = function docToEntity(doc) {
+  const baseEntity: Partial<Device & DeviceExpanded> = {
+    id: doc._id.toHexString(),
+    uid: doc.uid,
+    registered: doc.registered,
+    userAgent: doc.userAgent,
+    appVersion: doc.appVersion,
+    description: doc.description,
+  }
+  const userAttrs = doc.userId
+  if (userAttrs instanceof mongoose.Types.ObjectId) {
+    baseEntity.userId = userAttrs.toHexString()
+  }
+  else if (userAttrs) {
+    baseEntity.userId = userAttrs._id.toHexString()
+    baseEntity.user = userAttrs.displayName ? {
+      id: userAttrs._id.toHexString(),
+      displayName: userAttrs.displayName
+    } : null
+  }
+  return baseEntity as Device
+}
+
+export class DeviceMongooseRepository extends BaseMongooseRepository<DeviceDocument, DeviceModel, Device> implements DeviceRepository {
+
+  constructor(model: DeviceModel) {
+    super(model, { docToEntity })
+  }
+
+  async findByUid(uid: DeviceUid): Promise<Device | null> {
+    return await this.model.findOne({ uid }).then(x => x ? docToEntity(x) : null)
+  }
+  async findSome(findSpec: FindDevicesSpec): Promise<PageOf<DeviceExpanded>> {
+    const filter = dbFilterForFindSpec(findSpec)
+    const baseQuery = this.model.find(filter, null).lean()
+    const maybePopulateQuery = findSpec.expandUser ?
+      baseQuery.populate({
+        path: 'userId', select: 'displayName' ,
+        transform: (doc, _id) => doc ? doc : { _id }
+      }) :
+      baseQuery
+    const counted = await pageQuery(maybePopulateQuery, findSpec.paging)
+    const devices = Array.prototype.map.call(await counted.query, docToEntity) as DeviceExpanded[]
+    return pageOf(devices, findSpec.paging, counted.totalCount)
+  }
+  async countSome(findSpec: FindDevicesSpec): Promise<number> {
+    const filter = dbFilterForFindSpec(findSpec)
+    return await this.model.count(filter)
+  }
+}
+
+function dbFilterForFindSpec(findSpec: FindDevicesSpec): mongoose.FilterQuery<DeviceDocument> {
+  const filter: mongoose.FilterQuery<DeviceDocument> = {}
+  const where = findSpec.where
+  if (typeof where.containsSearchTerm === 'string') {
+    const searchRegex = new RegExp(_.escapeRegExp(where.containsSearchTerm), 'i')
+    filter.$or = [
+      { uid: searchRegex },
+      { userAgent: searchRegex },
+      { description: searchRegex },
+    ]
+  }
+  if (typeof where.registered === 'boolean') {
+    filter.registered = where.registered
+  }
+  return filter
+}
\ No newline at end of file
diff --git a/service/src/entities/devices/entities.devices.ts b/service/src/entities/devices/entities.devices.ts
index de5e7f11d..b8439b41a 100644
--- a/service/src/entities/devices/entities.devices.ts
+++ b/service/src/entities/devices/entities.devices.ts
@@ -1,4 +1,4 @@
-import { PagingParameters } from '../entities.global'
+import { PageOf, PagingParameters } from '../entities.global'
 import { User, UserId } from '../users/entities.users'
 
 export type DeviceId = string
@@ -11,15 +11,37 @@ export interface Device {
    */
   uid: DeviceUid
   description?: string | undefined
+  /**
+   * A registered device is one that has received approval from an administrator to access the Mage server.
+   */
   registered: boolean
-  userId: UserId
+  /**
+   * The device's `userId` is the user that owns the device.  If the user ID is null, multiple users may access Mage
+   * with the device.
+   */
+  userId?: UserId | null
   userAgent?: string | undefined
   appVersion?: string | undefined
 }
 
-export type DeviceExpanded = Device & { user: User | null }
+/**
+ * The related user entry will have only the user's ID and display name, but no other attributes.
+ */
+export type DeviceExpanded = Device & { user: Pick<User, 'id' | 'displayName'> | null }
 
 export interface FindDevicesSpec {
+  where: {
+    /**
+     * If present and `true` or `false`, match only devices with that value for the `registered` property.  Otherwise,
+     * match any device regardless of registered status.
+     */
+    registered?: boolean | null | undefined
+    /**
+     * Match only devices whose `userAgent`, `description`, or `uid` contain the given search term.
+     */
+    containsSearchTerm?: string | null | undefined
+  }
+  expandUser?: boolean | undefined
   paging: PagingParameters
 }
 
@@ -27,9 +49,9 @@ export interface DeviceRepository {
   create(deviceAttrs: Omit<Device, 'id'>): Promise<Device>
   update(deviceAttrs: Partial<Device>): Promise<Device | null>
   removeById(id: DeviceId): Promise<Device | null>
-  removeByUid(uid: DeviceUid): Promise<Device | null>
-  findById(id: DeviceId): Promise<null | DeviceExpanded>
-  findByUid(uid: DeviceUid): Promise<null | DeviceExpanded>
-  findSome(findSpec: FindDevicesSpec): Promise<Device[]>
+  findById(id: DeviceId): Promise<null | Device>
+  findByUid(uid: DeviceUid): Promise<null | Device>
+  findSome(findSpec: FindDevicesSpec & { userExpanded: false | undefined | never }): Promise<PageOf<Device>>
+  findSome(findSpec: FindDevicesSpec & { userExpanded: true }): Promise<PageOf<DeviceExpanded>>
   countSome(findSpec: FindDevicesSpec): Promise<number>
 }
\ No newline at end of file

From 3ededc07573ff13ca763ddbeec12fb14ff274065 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 16 Aug 2024 11:05:05 -0600
Subject: [PATCH 048/183] style(service): paging api code style

---
 service/src/entities/entities.global.ts | 52 +++++++++----------------
 1 file changed, 19 insertions(+), 33 deletions(-)

diff --git a/service/src/entities/entities.global.ts b/service/src/entities/entities.global.ts
index 8b637191e..af69592f3 100644
--- a/service/src/entities/entities.global.ts
+++ b/service/src/entities/entities.global.ts
@@ -76,53 +76,39 @@ export interface PagingParameters {
 }
 
 export interface PageOf<T> {
-  totalCount: number | null;
-  pageSize: number;
-  pageIndex: number;
-  items: T[];
+  totalCount: number | null
+  pageSize: number
+  pageIndex: number
+  items: T[]
   links?: {
-    next: number | null;
-    prev: number | null;
-  };
+    next: number | null
+    prev: number | null
+  }
 }
 
 export interface Links {
-  next: number | null;
-  prev: number | null;
+  next: number | null
+  prev: number | null
 }
 
-export function calculateLinks( paging: PagingParameters, totalCount: number | null): Links {
-  const links: Links = {
-    next: null,
-    prev: null
-  };
-
-  const limit = paging.pageSize;
-  const start = paging.pageIndex * limit;
-
-  if (start + limit < (totalCount || 0)) {
-    links.next = paging.pageIndex + 1;
-  }
-
-  if (start > 0) {
-    links.prev = Math.max(0, paging.pageIndex - 1);
-  }
-
-  return links;
+export function calculateLinks(paging: PagingParameters, totalCount: number | null): Links {
+  const limit = paging.pageSize
+  const start = paging.pageIndex * limit
+  const next = start + paging.pageSize < (totalCount || 0) ? paging.pageIndex + 1 : null
+  const prev = paging.pageIndex > 0 ? paging.pageIndex - 1 : null
+  return { next, prev }
 }
 
 export const pageOf = <T>(items: T[], paging: PagingParameters, totalCount?: number | null): PageOf<T> => {
-  // Provide a default value for totalCount if it's undefined
   const resolvedTotalCount = totalCount || 0;
-  const links = calculateLinks(paging, resolvedTotalCount);
-
+  const links = calculateLinks(paging, resolvedTotalCount)
   return {
-    totalCount: typeof totalCount === 'number' ? totalCount : null,
-    pageSize: paging.pageSize,
+    totalCount: totalCount || null,
+    pageSize: typeof paging.pageSize === 'number' ? paging.pageSize : -1,
     pageIndex: paging.pageIndex,
     items,
     links
-  };
+  }
 }
 
 /**

From 5b75495d3ccc459fffe3d31d682e515ef887a50c Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 16 Aug 2024 11:06:38 -0600
Subject: [PATCH 049/183] refactor(service): users-next: export payload type
 from toke verification module

---
 service/src/authentication/verification.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/authentication/verification.ts b/service/src/authentication/verification.ts
index 236d0d922..63175a786 100644
--- a/service/src/authentication/verification.ts
+++ b/service/src/authentication/verification.ts
@@ -1,7 +1,7 @@
 import JWT from 'jsonwebtoken'
 import { Json } from '../entities/entities.json_types'
 
-type Payload = {
+export type Payload = {
   subject: string | null,
   assertion: TokenAssertion | null,
   expiration: number | null,

From 9ccb2de167fab5831e831a46e3905587c8ae2339 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 16 Aug 2024 11:08:10 -0600
Subject: [PATCH 050/183] docs(service): add doc comment to query paging
 utility

---
 service/src/adapters/base/adapters.base.db.mongoose.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/service/src/adapters/base/adapters.base.db.mongoose.ts b/service/src/adapters/base/adapters.base.db.mongoose.ts
index 3401966e4..2111bff10 100644
--- a/service/src/adapters/base/adapters.base.db.mongoose.ts
+++ b/service/src/adapters/base/adapters.base.db.mongoose.ts
@@ -121,6 +121,12 @@ export class BaseMongooseRepository<DocType extends mongoose.AnyObject, Model ex
   }
 }
 
+/**
+ * Transform the given query into a paging query.  If {@link PagingParameters.includeTotalCount `includeTotalCount`}
+ * is `true`, or if `includeTotalCount` is `undefined` or `null`, and the requested page is the first page, execute
+ * a count operation before returning the paged query.  Return an object with entries for the `totalCount` and the
+ * query with the appropriate limit and skip operations applied for the requested page.
+ */
 export const pageQuery = <RT, DT>(query: mongoose.Query<RT, DT>, paging: PagingParameters): Promise<{ totalCount: number | null, query: mongoose.Query<RT, DT> }> => {
   const BaseQuery = query.toConstructor()
   const pageQuery = new BaseQuery().limit(paging.pageSize).skip(paging.pageIndex * paging.pageSize) as mongoose.Query<RT, DT>

From 0a82f4ffbad8ba2233b148a23aa4c9c8c4e5db1d Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 16 Aug 2024 16:02:34 -0600
Subject: [PATCH 051/183] refactor(service): devices typescript/di: add test
 placeholder for device mongoose repository

---
 .../adapters/devices/adapters.devices.mongoose.test.ts    | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 service/test/adapters/devices/adapters.devices.mongoose.test.ts

diff --git a/service/test/adapters/devices/adapters.devices.mongoose.test.ts b/service/test/adapters/devices/adapters.devices.mongoose.test.ts
new file mode 100644
index 000000000..4b5aabf33
--- /dev/null
+++ b/service/test/adapters/devices/adapters.devices.mongoose.test.ts
@@ -0,0 +1,8 @@
+import { expect } from 'chai'
+
+describe('device mongoose repository', function() {
+
+  it('has tests', function() {
+    expect.fail('todo')
+  })
+})
\ No newline at end of file

From bb77eb415a5fe6de25f21e7fbdb55cebfbd17277 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 16 Aug 2024 16:06:44 -0600
Subject: [PATCH 052/183] refactor(service): devices typescript/di: rename
 device routes to adapters typescript file

---
 .../devices/adapters.devices.controllers.web.ts}                  | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/src/{routes/devices.js => adapters/devices/adapters.devices.controllers.web.ts} (100%)

diff --git a/service/src/routes/devices.js b/service/src/adapters/devices/adapters.devices.controllers.web.ts
similarity index 100%
rename from service/src/routes/devices.js
rename to service/src/adapters/devices/adapters.devices.controllers.web.ts

From 3edff9a6e78fa941239fe118b5387833b7b556e3 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 17 Aug 2024 09:16:21 -0600
Subject: [PATCH 053/183] refactor(service)!: devices typescript/di: remove
 deprecated legacy device registration POST route

---
 .../adapters.devices.controllers.web.ts       | 169 +++++-------------
 1 file changed, 47 insertions(+), 122 deletions(-)

diff --git a/service/src/adapters/devices/adapters.devices.controllers.web.ts b/service/src/adapters/devices/adapters.devices.controllers.web.ts
index da9e7cc47..cf9878d54 100644
--- a/service/src/adapters/devices/adapters.devices.controllers.web.ts
+++ b/service/src/adapters/devices/adapters.devices.controllers.web.ts
@@ -1,135 +1,60 @@
-const log = require('winston');
+import express from 'express'
+import { DevicePermissionService } from '../../app.api/devices/app.api.devices'
+import { DevicePermission } from '../../entities/authorization/entities.permissions'
+import { DeviceRepository } from '../../entities/devices/entities.devices'
+import { WebAppRequestFactory } from '../adapters.controllers.web'
+const log = require('winston')
 const Device = require('../models/device');
 const access = require('../access');
 const pageInfoTransformer = require('../transformers/pageinfo.js');
 
-function DeviceResource() {}
-
-module.exports = function(app, security) {
-
-  var passport = security.authentication.passport;
-  var resource = new DeviceResource(passport);
-
-  // DEPRECATED retain old routes as deprecated until next major version.
-  /**
-   * @deprecated
-   */
-  app.post('/api/devices',
-    function authenticate(req, res, next) {
-      log.warn('DEPRECATED - The /api/devices route will be removed in the next major version, please use /auth/{auth_strategy}/devices');
-      passport.authenticate('local', function(err, user) {
-        if (err) {
-          return next(err);
-        }
-        if (!user) {
-          return next('route');
-        }
-        req.login(user, function(err) {
-          next(err);
-        });
-      })(req, res, next);
-    },
-    async function(req, res, next) {
-      const newDevice = {
-        uid: req.param('uid'),
-        name: req.param('name'),
-        registered: false,
-        description: req.param('description'),
-        userAgent: req.headers['user-agent'],
-        appVersion: req.param('appVersion'),
-        userId: req.user.id
-      };
-      try {
-        let device = await Device.getDeviceByUid(newDevice.uid);
-        if (!device) {
-          device = await Device.createDevice(newDevice);
-        }
-        return res.json(device)
-      }
-      catch(err) {
-        next(err);
-      }
-      next(new Error(`unknown error registering device ${newDevice.uid} for user ${newDevice.userId}`));
-    }
-  );
-
-  /**
-   * Create a new device, requires CREATE_DEVICE role
-   *
-   * @deprecated Use /auth/{strategy}/authorize instead.
-   */
-  app.post('/api/devices',
-    passport.authenticate('bearer'),
-    resource.ensurePermission('CREATE_DEVICE'),
-    resource.parseDeviceParams,
-    resource.validateDeviceParams,
-    resource.create
-  );
-
-  app.get('/api/devices/count',
-    passport.authenticate('bearer'),
-    access.authorize('READ_DEVICE'),
-    resource.count
-  );
 
-  // get all devices
-  app.get('/api/devices',
-    passport.authenticate('bearer'),
-    access.authorize('READ_DEVICE'),
-    resource.getDevices
-  );
+export function DeviceRoutes(deviceRepo: DeviceRepository, permissions: DevicePermissionService, appRequestFactory: WebAppRequestFactory): express.Router {
 
-  // get device
-  // TODO: check for READ_USER also
-  app.get('/api/devices/:id',
-    passport.authenticate('bearer'),
+  const deviceResource = express.Router()
+
+  deviceResource.get('/count',
     access.authorize('READ_DEVICE'),
-    resource.getDevice
-  );
-
-  // Update a device
-  app.put('/api/devices/:id',
-    passport.authenticate('bearer'),
-    access.authorize('UPDATE_DEVICE'),
-    resource.parseDeviceParams,
-    resource.updateDevice
-  );
-
-  // Delete a device
-  app.delete('/api/devices/:id',
-    passport.authenticate('bearer'),
-    access.authorize('DELETE_DEVICE'),
-    resource.deleteDevice
-  );
-};
+    resource.count
+  )
 
-DeviceResource.prototype.ensurePermission = function(permission) {
+  // TODO: check for READ_USER also
+  deviceResource.route('/:id')
+    .get(
+      access.authorize('READ_DEVICE'),
+      resource.getDevice
+    )
+    .put(
+      access.authorize('UPDATE_DEVICE'),
+      resource.parseDeviceParams,
+      resource.updateDevice
+    )
+    .delete(
+      access.authorize('DELETE_DEVICE'),
+      resource.deleteDevice
+    )
+
+  deviceResource.route('/')
+    .post(
+      resource.ensurePermission('CREATE_DEVICE'),
+      resource.parseDeviceParams,
+      resource.validateDeviceParams,
+      resource.create
+    )
+    .get(
+      access.authorize('READ_DEVICE'),
+      resource.getDevices
+    )
+  return deviceResource
+}
+
+function ensurePermission(permission: DevicePermission): express.RequestHandler {
   return function(req, res, next) {
-    access.userHasPermission(req.user, permission) ? next() : res.sendStatus(403);
-  };
-};
-
-/**
- * TODO: this should return a 201 and a location header
- *
- * @deprecated Use /auth/{strategy}/authorize instead.
- */
-DeviceResource.prototype.create = function(req, res, next) {
-  console.warn("Calling deprecated function to create device.  Call authorize instead.");
-
-  // Automatically register any device created by an ADMIN
-  req.newDevice.registered = true;
-
-  Device.createDevice(req.newDevice)
-    .then(device => {
-      res.json(device);
-    })
-    .catch(err => { 
-      next(err)
-    });
-};
+    access.userHasPermission(req.user, permission) ? next() : res.sendStatus(403)
+  }
+}
 
-DeviceResource.prototype.count = function (req, res, next) {
+function count(req: express., res, next) {
   var filter = {};
 
   if(req.query) {

From c8cee20b23cab3be28058306c5e3ba7c94c94207 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sun, 18 Aug 2024 07:33:52 -0600
Subject: [PATCH 054/183] refactor(service): remove mongoose document extension
 from role document type

---
 service/src/models/role.d.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/service/src/models/role.d.ts b/service/src/models/role.d.ts
index 138099520..f6784fa6a 100644
--- a/service/src/models/role.d.ts
+++ b/service/src/models/role.d.ts
@@ -1,10 +1,9 @@
 
-import mongoose from 'mongoose'
 import { AnyPermission } from '../entities/authorization/entities.permissions'
 
 type Callback<R> = (err: any, result?: R) => any
 
-export declare interface RoleDocument extends mongoose.Document {
+export declare interface RoleDocument {
   id: string
   name: string
   description?: string

From 8c68e53e716a71813222ea6e1505dcf8ca2abb7e Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sun, 18 Aug 2024 07:35:39 -0600
Subject: [PATCH 055/183] refactor(service): remove old authentication entity
 types

---
 .../authentication/entities.authentication.ts      | 14 --------------
 1 file changed, 14 deletions(-)
 delete mode 100644 service/src/entities/authentication/entities.authentication.ts

diff --git a/service/src/entities/authentication/entities.authentication.ts b/service/src/entities/authentication/entities.authentication.ts
deleted file mode 100644
index aae1fd6ac..000000000
--- a/service/src/entities/authentication/entities.authentication.ts
+++ /dev/null
@@ -1,14 +0,0 @@
-export interface Authentication {
-  id: string
-  type: string
-}
-
-/**
- * TODO: move somewhere else
- */
-export interface SecurityStatus {
-  locked: boolean
-  lockedUntil: Date
-  invalidLoginAttempts: number
-  numberOfTimesLocked: number
-}

From 8842c42da5c5907af1804d2c84a0474805daa015 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sun, 18 Aug 2024 08:40:51 -0600
Subject: [PATCH 056/183] refactor(service)!: devices typescript/di: add device
 entities and permission types

---
 .../src/app.api/devices/app.api.devices.ts    |  9 ++++++++
 .../authentication/entities.authentication.ts | 22 +++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 service/src/app.api/devices/app.api.devices.ts
 create mode 100644 service/src/authentication/entities.authentication.ts

diff --git a/service/src/app.api/devices/app.api.devices.ts b/service/src/app.api/devices/app.api.devices.ts
new file mode 100644
index 000000000..7162675b9
--- /dev/null
+++ b/service/src/app.api/devices/app.api.devices.ts
@@ -0,0 +1,9 @@
+import { PermissionDeniedError } from '../app.api.errors'
+import { AppRequestContext } from '../app.api.global'
+
+export interface DevicePermissionService {
+  ensureCreateDevicePermission(context: AppRequestContext): Promise<null | PermissionDeniedError>
+  ensureReadDevicePermission(context: AppRequestContext): Promise<null | PermissionDeniedError>
+  ensureUpdateDevicePermission(context: AppRequestContext): Promise<null | PermissionDeniedError>
+  ensureDeleteDevicePermission(context: AppRequestContext): Promise<null | PermissionDeniedError>
+}
\ No newline at end of file
diff --git a/service/src/authentication/entities.authentication.ts b/service/src/authentication/entities.authentication.ts
new file mode 100644
index 000000000..370e9ecc4
--- /dev/null
+++ b/service/src/authentication/entities.authentication.ts
@@ -0,0 +1,22 @@
+import { Device, DeviceId } from '../entities/devices/entities.devices'
+import { UserExpanded, UserId } from '../entities/users/entities.users'
+
+export interface Session {
+  token: string
+  expirationDate: Date
+  user: UserId
+  device?: DeviceId | null
+}
+
+export type SessionExpanded = Omit<Session, 'user' | 'device'> & {
+  user: UserExpanded
+  device: Device
+}
+
+export interface SessionRepository {
+  findSessionByToken(token: string): Promise<Session | null>
+  createOrRefreshSession(userId: UserId, deviceId?: string): Promise<Session>
+  removeSession(token: string): Promise<void>
+  removeSessionsForUser(userId: UserId): Promise<number>
+  removeSessionForDevice(deviceId: DeviceId): Promise<number>
+}
\ No newline at end of file

From 264bb74e8e0b64c32541930b0f39af148ff5a28f Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 20 Aug 2024 14:53:59 -0600
Subject: [PATCH 057/183] refactor(service): devices typescript/di: implement
 device web controller

---
 .../adapters.devices.controllers.web.ts       | 300 ++++++++++--------
 .../devices/adapters.devices.db.mongoose.ts   |   2 +
 .../src/entities/devices/entities.devices.ts  |  14 +-
 service/src/models/device.js                  |   2 +
 4 files changed, 191 insertions(+), 127 deletions(-)

diff --git a/service/src/adapters/devices/adapters.devices.controllers.web.ts b/service/src/adapters/devices/adapters.devices.controllers.web.ts
index cf9878d54..00a7e7620 100644
--- a/service/src/adapters/devices/adapters.devices.controllers.web.ts
+++ b/service/src/adapters/devices/adapters.devices.controllers.web.ts
@@ -1,162 +1,214 @@
 import express from 'express'
+import { entityNotFound, invalidInput } from '../../app.api/app.api.errors'
 import { DevicePermissionService } from '../../app.api/devices/app.api.devices'
-import { DevicePermission } from '../../entities/authorization/entities.permissions'
-import { DeviceRepository } from '../../entities/devices/entities.devices'
-import { WebAppRequestFactory } from '../adapters.controllers.web'
+import { Device, DeviceRepository, FindDevicesSpec } from '../../entities/devices/entities.devices'
+import { PageOf, PagingParameters } from '../../entities/entities.global'
+import { User, UserFindParameters, UserRepository } from '../../entities/users/entities.users'
+import { compatibilityMageAppErrorHandler, WebAppRequestFactory } from '../adapters.controllers.web'
 const log = require('winston')
-const Device = require('../models/device');
-const access = require('../access');
-const pageInfoTransformer = require('../transformers/pageinfo.js');
 
 
-export function DeviceRoutes(deviceRepo: DeviceRepository, permissions: DevicePermissionService, appRequestFactory: WebAppRequestFactory): express.Router {
+export function DeviceRoutes(deviceRepo: DeviceRepository, userRepo: UserRepository, permissions: DevicePermissionService, createAppContext: WebAppRequestFactory): express.Router {
 
   const deviceResource = express.Router()
+  const ensurePermission = PermissionMiddleware(permissions, createAppContext)
 
   deviceResource.get('/count',
-    access.authorize('READ_DEVICE'),
-    resource.count
+    ensurePermission.create,
+    async (req, res, next) => {
+      const findSpec = parseDeviceFindSpec(req)
+      try {
+        const count = await deviceRepo.countSome(findSpec)
+        return res.json({ count })
+      }
+      catch (err) {
+        next(err)
+      }
+    }
   )
 
   // TODO: check for READ_USER also
+  // ... meh
   deviceResource.route('/:id')
     .get(
-      access.authorize('READ_DEVICE'),
-      resource.getDevice
+      ensurePermission.read,
+      async (req, res, next) => {
+        const id = req.params.id
+        try {
+          const device = deviceRepo.findById(id)
+          if (device) {
+            return res.json(device)
+          }
+        }
+        catch (err) {
+          next(err)
+        }
+      }
     )
     .put(
-      access.authorize('UPDATE_DEVICE'),
-      resource.parseDeviceParams,
-      resource.updateDevice
+      ensurePermission.update,
+      express.urlencoded(),
+      async (req, res, next) => {
+        const idInPath = req.params.id
+        const update = parseDeviceAttributes(req)
+        if (typeof update.id === 'string' && update.id !== idInPath) {
+          return next(invalidInput(`body id ${update.id} does not match id in path ${idInPath}`))
+        }
+        try {
+          const updated = await deviceRepo.update({ ...update, id: idInPath })
+          if (updated) {
+            return res.json(updated)
+          }
+          return next(entityNotFound(idInPath, 'Device'))
+        }
+        catch (err) {
+          next(err)
+        }
+      }
     )
     .delete(
-      access.authorize('DELETE_DEVICE'),
-      resource.deleteDevice
+      ensurePermission.delete,
+      async (req, res, next) => {
+        try {
+          const idInPath = req.params.id
+          const deleted = await deviceRepo.removeById(idInPath)
+          if (deleted) {
+            return res.json(deleted)
+          }
+          return next(entityNotFound(idInPath, 'Device'))
+        }
+        catch (err) {
+          next(err)
+        }
+      }
     )
 
   deviceResource.route('/')
     .post(
-      resource.ensurePermission('CREATE_DEVICE'),
-      resource.parseDeviceParams,
-      resource.validateDeviceParams,
-      resource.create
+      ensurePermission.create,
+      express.urlencoded(),
+      async (req, res, next) => {
+        const attrs = parseDeviceAttributes(req)
+        if (typeof attrs.uid !== 'string') {
+          return res.status(400).send('missing uid')
+        }
+        if (typeof attrs.registered !== 'boolean') {
+          attrs.registered = false
+        }
+        try {
+          const device = await deviceRepo.create(attrs as Device)
+          return res.status(201).json(device)
+        }
+        catch (err) {
+          next(err)
+        }
+      }
     )
     .get(
-      access.authorize('READ_DEVICE'),
-      resource.getDevices
+      ensurePermission.read,
+      async (req, res, next) => {
+        const findDevices = { ...parseDeviceFindSpec(req), expandUser: true } as FindDevicesSpec & { expandUser: true }
+        const userSearch: UserFindParameters | null = typeof findDevices.where.containsSearchTerm === 'string' ?
+          { nameOrContactTerm: findDevices.where.containsSearchTerm, pageIndex: 0, pageSize: Number.MAX_SAFE_INTEGER } :
+          null
+        try {
+          // TODO: would this user query be necessary if the web app's user page just showed the user's devices?
+          const usersPage: PageOf<User> = userSearch ?
+            await userRepo.find(userSearch) :
+            { pageIndex: 0, pageSize: 0, totalCount: null, items: [] }
+          const userIds = usersPage.items.map(x => x.id)
+          // TODO: hope user id list is not too big
+          if (userIds.length) {
+            findDevices.where.userIdIsAnyOf = userIds
+          }
+          const devicesPage = await deviceRepo.findSome(findDevices)
+          const { items, ...paging } = devicesPage
+          const compatibilityDevicesPage = {
+            count: paging.totalCount,
+            // web app really only uses the links entry
+            paginInfo: paging,
+            devices: items
+          }
+          return res.json(compatibilityDevicesPage)
+        }
+        catch (err) {
+          next(err)
+        }
+      }
     )
-  return deviceResource
-}
 
-function ensurePermission(permission: DevicePermission): express.RequestHandler {
-  return function(req, res, next) {
-    access.userHasPermission(req.user, permission) ? next() : res.sendStatus(403)
-  }
+  return deviceResource.use(compatibilityMageAppErrorHandler)
 }
 
-function count(req: express., res, next) {
-  var filter = {};
 
-  if(req.query) {
-    for (let [key, value] of Object.entries(req.query)) {
-      if(key == 'populate' || key == 'limit' || key == 'start' || key == 'sort' || key == 'forceRefresh'){
-        continue;
-      }
-      filter[key] = value;
-    }
-  }
-
-  Device.count({ filter: filter })
-    .then(count => res.json({count: count}))
-    .catch(err => next(err));
-};
-
-/**
- * TODO:
- * * the /users route uses the `populate` query param while this uses
- *   `expand`; should be consistent
- * * openapi supports array query parameters using the pipe `|` delimiter;
- *   use that instead of comma for the `expand` query param. on the other hand,
- *   this only actually supports a singular `expand` key, so why bother with
- *   the split anyway?
- */
-DeviceResource.prototype.getDevices = function (req, res, next) {
 
-  const { populate, expand, limit, start, sort, forceRefresh, ...filter } = req.query;
-  const expandFlags = { user: /\buser\b/i.test(expand) };
-
-  Device.getDevices({ filter, expand: expandFlags, limit, start, sort })
-    .then(result => {
-      if (!Array.isArray(result)) {
-        result = pageInfoTransformer.transform(result, req);
-      }
-      return res.json(result);
-    })
-    .catch(err => next(err));
-};
+interface PermissionMiddleware {
+  create: express.RequestHandler
+  read: express.RequestHandler
+  update: express.RequestHandler
+  delete: express.RequestHandler
+}
 
-DeviceResource.prototype.getDevice = function(req, res, next) {
-  var expand = {};
-  if (req.query.expand) {
-    var expandList = req.query.expand.split(",");
-    if (expandList.some(function(e) { return e === 'user';})) {
-      expand.user = true;
+function PermissionMiddleware(permissionService: DevicePermissionService, createAppContext: WebAppRequestFactory): PermissionMiddleware {
+  return {
+    async create(req, res, next): Promise<void> {
+      next(await permissionService.ensureCreateDevicePermission(createAppContext(req)))
+    },
+    async read(req, res, next): Promise<void> {
+      next(await permissionService.ensureReadDevicePermission(createAppContext(req)))
+    },
+    async update(req, res, next): Promise<void> {
+      next(await permissionService.ensureUpdateDevicePermission(createAppContext(req)))
+    },
+    async delete(req, res, next): Promise<void> {
+      next(await permissionService.ensureDeleteDevicePermission(createAppContext(req)))
     }
   }
+}
 
-  Device.getDeviceById(req.params.id, {expand: expand})
-    .then(device => res.json(device))
-    .catch(err => next(err));
-};
-
-DeviceResource.prototype.updateDevice = function(req, res, next) {
-  const update = {};
-  if (req.newDevice.uid) update.uid = req.newDevice.uid;
-  if (req.newDevice.name) update.name = req.newDevice.name;
-  if (req.newDevice.description) update.description = req.newDevice.description;
-  if (req.newDevice.registered !== undefined) update.registered = req.newDevice.registered;
-  if (req.newDevice.userId) update.userId = req.newDevice.userId;
-
-  Device.updateDevice(req.param('id'), update)
-    .then(device => {
-      if (!device) return res.sendStatus(404);
-
-      res.json(device);
-    })
-    .catch(err => {
-      next(err)
-    });
-};
-
-DeviceResource.prototype.deleteDevice = function(req, res, next) {
-  Device.deleteDevice(req.param('id'))
-    .then(device => {
-      if (!device) return res.sendStatus(404);
-
-      res.json(device);
-    })
-    .catch(err => next(err));
-};
-
-DeviceResource.prototype.parseDeviceParams = function(req, res, next) {
-  req.newDevice = {
-    uid: req.param('uid'),
-    name: req.param('name'),
-    description: req.param('description'),
-    userId: req.param('userId')
-  };
-
-  if (req.param('registered') !== undefined) {
-    req.newDevice.registered = req.param('registered') === 'true';
+function parseDeviceAttributes(req: express.Request): Partial<Device> {
+  const { uid, name, description, userId, registered } = req.body
+  const attrs: Record<string, any> = {}
+  if (typeof uid === 'string') {
+    attrs.uid = uid
   }
+  if (typeof name === 'string') {
+    attrs.name = name
+  }
+  if (typeof description === 'string') {
+    attrs.description = description
+  }
+  if (typeof userId === 'string') {
+    attrs.userId = userId
+  }
+  if (typeof registered === 'boolean') {
+    attrs.registered = registered
+  }
+  return attrs
+}
 
-  next();
-};
-
-DeviceResource.prototype.validateDeviceParams = function(req, res, next) {
-  if (!req.newDevice.uid) {
-    return res.status(400).send("missing required param 'uid'");
+function parseDeviceFindSpec(req: express.Request): FindDevicesSpec {
+  const { registered, search, limit, start } = req.query
+  const filter: FindDevicesSpec['where'] = {}
+  if (typeof registered === 'string') {
+    const lowerRegistered = registered.toLowerCase()
+    if (lowerRegistered === 'true') {
+      filter.registered = true
+    }
+    else if (lowerRegistered === 'false') {
+      filter.registered = false
+    }
+  }
+  if (typeof search === 'string' && search.length > 0) {
+    filter.containsSearchTerm = search
   }
+  const limitParsed = parseInt(String(limit)) || Number.MAX_SAFE_INTEGER
+  const startParsed = parseInt(String(start)) || 0
+  const paging: PagingParameters = {
+    pageIndex: startParsed,
+    pageSize: limitParsed,
+  }
+  const spec: FindDevicesSpec = { where: filter, paging }
+  return spec
+}
 
-  next();
-};
diff --git a/service/src/adapters/devices/adapters.devices.db.mongoose.ts b/service/src/adapters/devices/adapters.devices.db.mongoose.ts
index 8d6f5731f..a8c1b7364 100644
--- a/service/src/adapters/devices/adapters.devices.db.mongoose.ts
+++ b/service/src/adapters/devices/adapters.devices.db.mongoose.ts
@@ -63,6 +63,7 @@ export class DeviceMongooseRepository extends BaseMongooseRepository<DeviceDocum
   async findByUid(uid: DeviceUid): Promise<Device | null> {
     return await this.model.findOne({ uid }).then(x => x ? docToEntity(x) : null)
   }
+
   async findSome(findSpec: FindDevicesSpec): Promise<PageOf<DeviceExpanded>> {
     const filter = dbFilterForFindSpec(findSpec)
     const baseQuery = this.model.find(filter, null).lean()
@@ -76,6 +77,7 @@ export class DeviceMongooseRepository extends BaseMongooseRepository<DeviceDocum
     const devices = Array.prototype.map.call(await counted.query, docToEntity) as DeviceExpanded[]
     return pageOf(devices, findSpec.paging, counted.totalCount)
   }
+
   async countSome(findSpec: FindDevicesSpec): Promise<number> {
     const filter = dbFilterForFindSpec(findSpec)
     return await this.model.count(filter)
diff --git a/service/src/entities/devices/entities.devices.ts b/service/src/entities/devices/entities.devices.ts
index b8439b41a..036475264 100644
--- a/service/src/entities/devices/entities.devices.ts
+++ b/service/src/entities/devices/entities.devices.ts
@@ -40,18 +40,26 @@ export interface FindDevicesSpec {
      * Match only devices whose `userAgent`, `description`, or `uid` contain the given search term.
      */
     containsSearchTerm?: string | null | undefined
+    /**
+     * Match devices whose `userId` is in the given array.  This constraint is independent of `containsSearchTerm`, but
+     * subject to the `registered` constraint.
+     */
+    userIdIsAnyOf?: UserId[]
   }
+  /**
+   * If `true`, return {@link DeviceExpanded results}
+   */
   expandUser?: boolean | undefined
   paging: PagingParameters
 }
 
 export interface DeviceRepository {
   create(deviceAttrs: Omit<Device, 'id'>): Promise<Device>
-  update(deviceAttrs: Partial<Device>): Promise<Device | null>
+  update(deviceAttrs: Partial<Device> & Pick<Device, 'id'>): Promise<Device | null>
   removeById(id: DeviceId): Promise<Device | null>
   findById(id: DeviceId): Promise<null | Device>
   findByUid(uid: DeviceUid): Promise<null | Device>
-  findSome(findSpec: FindDevicesSpec & { userExpanded: false | undefined | never }): Promise<PageOf<Device>>
-  findSome(findSpec: FindDevicesSpec & { userExpanded: true }): Promise<PageOf<DeviceExpanded>>
+  findSome(findSpec: FindDevicesSpec & { expandUser: false | undefined | never }): Promise<PageOf<Device>>
+  findSome(findSpec: FindDevicesSpec & { expandUser: true }): Promise<PageOf<DeviceExpanded>>
   countSome(findSpec: FindDevicesSpec): Promise<number>
 }
\ No newline at end of file
diff --git a/service/src/models/device.js b/service/src/models/device.js
index 9a2e1d6fa..965abfa45 100644
--- a/service/src/models/device.js
+++ b/service/src/models/device.js
@@ -43,6 +43,7 @@ DeviceSchema.path('userId').validate(async function (userId) {
 
 DeviceSchema.pre('findOneAndUpdate', function (next) {
   if (this.getUpdate() && this.getUpdate().registered === false) {
+    // TODO: users-next: new token methods
     Token.removeTokenForDevice({ _id: this.getQuery()._id }, function (err) {
       next(err);
     });
@@ -57,6 +58,7 @@ DeviceSchema.pre('findOneAndDelete', function (next) {
 
   async.parallel({
     token: function (done) {
+      // TODO: users-next: new token methods
       Token.removeTokenForDevice({ _id: query.getQuery()._id }, function (err) {
         done(err);
       });

From 05da6909e20a7303ea9c00314dc96af5c7b6fca2 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 20 Aug 2024 15:34:56 -0600
Subject: [PATCH 058/183] refactor(web-app): devices typescript/di: remove
 unnecessary sort parameter from device paging service as nothing ever changes
 the sort order

---
 web-app/src/ng1/factories/device-paging.service.js | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/web-app/src/ng1/factories/device-paging.service.js b/web-app/src/ng1/factories/device-paging.service.js
index eda141213..598dca99e 100644
--- a/web-app/src/ng1/factories/device-paging.service.js
+++ b/web-app/src/ng1/factories/device-paging.service.js
@@ -22,21 +22,21 @@ function DevicePagingService(DeviceService, $q) {
         return {
             all: {
                 countFilter: {},
-                deviceFilter: { limit: 10, sort: { userAgent: 1, _id: 1 } },
+                deviceFilter: { limit: 10 },
                 searchFilter: '',
                 deviceCount: 0,
                 pageInfo: {}
             },
             registered: {
                 countFilter: { registered: true },
-                deviceFilter: { limit: 10, sort: { userAgent: 1, _id: 1 }, registered: true },
+                deviceFilter: { limit: 10, registered: true },
                 searchFilter: '',
                 deviceCount: 0,
                 pageInfo: {}
             },
             unregistered: {
                 countFilter: { registered: false },
-                deviceFilter: { limit: 10, sort: { userAgent: 1, _id: 1 }, registered: false },
+                deviceFilter: { limit: 10, registered: false },
                 searchFilter: '',
                 deviceCount: 0,
                 pageInfo: {}
@@ -153,15 +153,12 @@ function DevicePagingService(DeviceService, $q) {
             promise = $q.resolve(data.pageInfo.devices);
         } else {
             //Perform the server side searching
+            // TODO: use /next-users/search
             data.searchFilter = deviceSearch;
 
             var filter = data.deviceFilter;
             if (userSearch == null) {
-                filter.or = {
-                    userAgent: '.*' + deviceSearch + '.*',
-                    description: '.*' + deviceSearch + '.*',
-                    uid:  '.*' + deviceSearch + '.*'
-                };
+                filter.search = deviceSearch
             } else {
                 filter.or = {
                     displayName: '.*' + userSearch + '.*',

From aaeb23ceaba6cdcb5173e87bc9814b147058c04e Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 20 Aug 2024 15:49:40 -0600
Subject: [PATCH 059/183] refactor(service): devices typescript/di: add user id
 constraint to mongoose device repository device query

---
 .../devices/adapters.devices.db.mongoose.ts         | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/service/src/adapters/devices/adapters.devices.db.mongoose.ts b/service/src/adapters/devices/adapters.devices.db.mongoose.ts
index a8c1b7364..8f4761dc1 100644
--- a/service/src/adapters/devices/adapters.devices.db.mongoose.ts
+++ b/service/src/adapters/devices/adapters.devices.db.mongoose.ts
@@ -66,11 +66,11 @@ export class DeviceMongooseRepository extends BaseMongooseRepository<DeviceDocum
 
   async findSome(findSpec: FindDevicesSpec): Promise<PageOf<DeviceExpanded>> {
     const filter = dbFilterForFindSpec(findSpec)
-    const baseQuery = this.model.find(filter, null).lean()
+    const baseQuery = this.model.find(filter, null, { sort: 'userAgent _id' }).lean()
     const maybePopulateQuery = findSpec.expandUser ?
       baseQuery.populate({
         path: 'userId', select: 'displayName' ,
-        transform: (doc, _id) => doc ? doc : { _id }
+        transform: (doc, _id) => (doc ? doc : { _id })
       }) :
       baseQuery
     const counted = await pageQuery(maybePopulateQuery, findSpec.paging)
@@ -95,6 +95,15 @@ function dbFilterForFindSpec(findSpec: FindDevicesSpec): mongoose.FilterQuery<De
       { description: searchRegex },
     ]
   }
+  if (Array.isArray(where.userIdIsAnyOf) && where.userIdIsAnyOf.length) {
+    const userIdFilter = { $in: where.userIdIsAnyOf }
+    if (Array.isArray(filter.$or)) {
+      filter.$or.push({ userId: userIdFilter })
+    }
+    else {
+      filter.userId = userIdFilter
+    }
+  }
   if (typeof where.registered === 'boolean') {
     filter.registered = where.registered
   }

From 0b931d530d0dd3ada323a944dac7f4badf2cc969 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 20 Aug 2024 23:06:45 -0600
Subject: [PATCH 060/183] refactor(service): devices typescript/di: remove
 associated sessions for removed device

---
 .../adapters/devices/adapters.devices.controllers.web.ts   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/service/src/adapters/devices/adapters.devices.controllers.web.ts b/service/src/adapters/devices/adapters.devices.controllers.web.ts
index 00a7e7620..08c66e089 100644
--- a/service/src/adapters/devices/adapters.devices.controllers.web.ts
+++ b/service/src/adapters/devices/adapters.devices.controllers.web.ts
@@ -1,14 +1,14 @@
 import express from 'express'
 import { entityNotFound, invalidInput } from '../../app.api/app.api.errors'
 import { DevicePermissionService } from '../../app.api/devices/app.api.devices'
+import { SessionRepository } from '../../authentication/entities.authentication'
 import { Device, DeviceRepository, FindDevicesSpec } from '../../entities/devices/entities.devices'
 import { PageOf, PagingParameters } from '../../entities/entities.global'
 import { User, UserFindParameters, UserRepository } from '../../entities/users/entities.users'
 import { compatibilityMageAppErrorHandler, WebAppRequestFactory } from '../adapters.controllers.web'
-const log = require('winston')
 
 
-export function DeviceRoutes(deviceRepo: DeviceRepository, userRepo: UserRepository, permissions: DevicePermissionService, createAppContext: WebAppRequestFactory): express.Router {
+export function DeviceRoutes(deviceRepo: DeviceRepository, userRepo: UserRepository, sessionRepo: SessionRepository, permissions: DevicePermissionService, createAppContext: WebAppRequestFactory): express.Router {
 
   const deviceResource = express.Router()
   const ensurePermission = PermissionMiddleware(permissions, createAppContext)
@@ -71,7 +71,10 @@ export function DeviceRoutes(deviceRepo: DeviceRepository, userRepo: UserReposit
       async (req, res, next) => {
         try {
           const idInPath = req.params.id
+          console.info(`delete device`, idInPath)
           const deleted = await deviceRepo.removeById(idInPath)
+          const removedSessionsCount = sessionRepo.removeSessionForDevice(idInPath)
+          console.info(`removed ${removedSessionsCount} session(s) for device ${idInPath}`)
           if (deleted) {
             return res.json(deleted)
           }

From 069400ee60d44479cfb2afe8c06c319f60fb0c55 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 20 Aug 2024 23:18:46 -0600
Subject: [PATCH 061/183] refactor(service): devices typescript/di: remove
 associated sessions for unregistered device

---
 .../adapters/devices/adapters.devices.controllers.web.ts    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/service/src/adapters/devices/adapters.devices.controllers.web.ts b/service/src/adapters/devices/adapters.devices.controllers.web.ts
index 08c66e089..641bed187 100644
--- a/service/src/adapters/devices/adapters.devices.controllers.web.ts
+++ b/service/src/adapters/devices/adapters.devices.controllers.web.ts
@@ -51,10 +51,16 @@ export function DeviceRoutes(deviceRepo: DeviceRepository, userRepo: UserReposit
       async (req, res, next) => {
         const idInPath = req.params.id
         const update = parseDeviceAttributes(req)
+        // TODO: if request is marking registered false, remove associated sessions like mongoose middleware
         if (typeof update.id === 'string' && update.id !== idInPath) {
           return next(invalidInput(`body id ${update.id} does not match id in path ${idInPath}`))
         }
         try {
+          if (update.registered === false) {
+            console.info(`update device ${idInPath} to unregistered`)
+            const sessionsRemovedCount = await sessionRepo.removeSessionForDevice(idInPath)
+            console.info(`removed ${sessionsRemovedCount} session(s) for device ${idInPath}`)
+          }
           const updated = await deviceRepo.update({ ...update, id: idInPath })
           if (updated) {
             return res.json(updated)

From f4198cb1423c02566177c6bc14dd4a174e1bc85a Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 21 Aug 2024 07:21:15 -0600
Subject: [PATCH 062/183] refactor(service): devices typescript/di: add todo
 comment

---
 .../src/adapters/devices/adapters.devices.controllers.web.ts    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/service/src/adapters/devices/adapters.devices.controllers.web.ts b/service/src/adapters/devices/adapters.devices.controllers.web.ts
index 641bed187..1e13635aa 100644
--- a/service/src/adapters/devices/adapters.devices.controllers.web.ts
+++ b/service/src/adapters/devices/adapters.devices.controllers.web.ts
@@ -81,6 +81,8 @@ export function DeviceRoutes(deviceRepo: DeviceRepository, userRepo: UserReposit
           const deleted = await deviceRepo.removeById(idInPath)
           const removedSessionsCount = sessionRepo.removeSessionForDevice(idInPath)
           console.info(`removed ${removedSessionsCount} session(s) for device ${idInPath}`)
+          // TODO: the old observation model had a middleware that removed the device id from created observations,
+          // but do we really care that much
           if (deleted) {
             return res.json(deleted)
           }

From aab463a88fff815137643a6aa7415f7d3548ee2d Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 21 Aug 2024 09:24:14 -0600
Subject: [PATCH 063/183] refactor(service): devices typescript/di: add
 functional test stubs for device operations

---
 .../functionalTests/devices/devices.test.ts   | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 service/functionalTests/devices/devices.test.ts

diff --git a/service/functionalTests/devices/devices.test.ts b/service/functionalTests/devices/devices.test.ts
new file mode 100644
index 000000000..c6f705a6a
--- /dev/null
+++ b/service/functionalTests/devices/devices.test.ts
@@ -0,0 +1,33 @@
+import { expect } from 'chai'
+
+describe('device operations', function() {
+
+  describe('removing', function() {
+
+    it('invalidates associated sessions', async function() {
+      expect.fail('todo')
+    })
+
+    it('prevents the owning user from authenticating with the device', async function() {
+      expect.fail('todo')
+    })
+  })
+
+  describe('disabling', function() {
+
+    it('invalidates existing associated sessions', async function() {
+      expect.fail('todo')
+    })
+
+    it('prevents the owning user from authenticating with the device', async function() {
+      expect.fail('todo')
+    })
+  })
+
+  describe('enabling', function() {
+
+    it('allows the owning user to authenticate with the device', async function() {
+      expect.fail('todo')
+    })
+  })
+})
\ No newline at end of file

From 73fd383d733ea3345b760479a17d2d15af4a96cd Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 21 Aug 2024 09:54:27 -0600
Subject: [PATCH 064/183] style(service): type lint error

---
 service/src/adapters/adapters.controllers.web.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/adapters/adapters.controllers.web.ts b/service/src/adapters/adapters.controllers.web.ts
index 3f95dd7ba..0ece6201b 100644
--- a/service/src/adapters/adapters.controllers.web.ts
+++ b/service/src/adapters/adapters.controllers.web.ts
@@ -3,7 +3,7 @@ import { ErrEntityNotFound, ErrInfrastructure, ErrInvalidInput, ErrPermissionDen
 import { AppRequest } from '../app.api/app.api.global'
 
 export interface WebAppRequestFactory<Req extends AppRequest = AppRequest> {
-  <RequestParams extends object = {}>(webReq: express.Request, params?: RequestParams): Req & RequestParams
+  <RequestParams extends object = object>(webReq: express.Request, params?: RequestParams): Req & RequestParams
 }
 
 /**

From 92fa8cb80fa19220da0f7e853600f6410ff3ce02 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 21 Aug 2024 10:50:45 -0600
Subject: [PATCH 065/183] chore(service): changelog updates

---
 CHANGELOG.md | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 823def7de..d2001a6ef 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,12 +17,23 @@ MAGE adheres to [Semantic Versioning](http://semver.org/).
 - The `MAGE_MONGO_TLS_INSECURE` env var avoids issues with [self-signed certs](https://github.com/Automattic/mongoose/issues/9147).
 - [GARS](https://github.com/ngageoint/gars-js) grid overlay
 - [MGRS](https://github.com/ngageoint/mgrs-js) grid overlay
+- Add support for sorting observations by `timestamp`: `GET /api/observations?sort=timestamp+(asc|desc)`
 
 ##### Bug fixes
 - Single observation download bug
 - Protect against disabling all authentications.
 - Problem with OAuth web login
 
+##### Benign API Changes
+- `/api/events/{eventId}/observations`
+  - Remove support for `geometry` query parameter
+  - Remove support for `fields` query parameter
+- `/api/events/{eventId}/observations/{observationId}`
+  - Remove support for unnecessary observation query parameters
+- `/api/devices`
+  - Remove support for `sort` query parameter
+  - Remove support for `expand` query parameter
+
 ## [6.2.12](https://github.com/ngageoint/mage-server/releases/tag/6.2.12)
 ### Service
 #### Security

From f763f998b572e2e78a94b37ab61bec6bd110447d Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 21 Aug 2024 10:52:34 -0600
Subject: [PATCH 066/183] refactor(service): users typescript/di: add todo
 comment

---
 service/src/api/user.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/service/src/api/user.js b/service/src/api/user.js
index d13407939..26028f6a3 100644
--- a/service/src/api/user.js
+++ b/service/src/api/user.js
@@ -52,6 +52,10 @@ User.prototype.login = function (user, device, options, callback) {
 
       if (device) {
         // set user-agent and mage version on device
+        /*
+        TODO: users-next: don't do this - set user agent on login record and leave user agent as registered on device
+        record maybe even require re-approval if configured device policy dictates
+        */
         DeviceModel.updateDevice(device._id, { userAgent: options.userAgent, appVersion: options.appVersion }).then(() => {
           callback(null, token);
         }).catch(err => {

From f637dfa2b3982436a27bdcd143de630552bab89b Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 21 Aug 2024 11:29:24 -0600
Subject: [PATCH 067/183] refactor(service): users typescript/di: move device
 functional tests to security folder

---
 .../devices.test.ts => security/devices.security.test.ts}         | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/functionalTests/{devices/devices.test.ts => security/devices.security.test.ts} (100%)

diff --git a/service/functionalTests/devices/devices.test.ts b/service/functionalTests/security/devices.security.test.ts
similarity index 100%
rename from service/functionalTests/devices/devices.test.ts
rename to service/functionalTests/security/devices.security.test.ts

From 448d8d3bad94ea40b00900a711449b1d7e3e4cea Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 21 Aug 2024 14:24:03 -0600
Subject: [PATCH 068/183] refactor(service): users typescript/di: remove
 unreferenced factory function from upload middleware export

---
 service/src/upload.ts | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/service/src/upload.ts b/service/src/upload.ts
index 2e6f9e22b..85ee97831 100644
--- a/service/src/upload.ts
+++ b/service/src/upload.ts
@@ -6,26 +6,21 @@ import env from './environment/env'
 
 const storage = multer.diskStorage({
   destination: env.tempDirectory,
-  filename: function(req, file, cb: any) {
+  filename: function(req, file, cb: (error: Error | null, filename: string) => void): void {
     crypto.pseudoRandomBytes(16, function(err, raw) {
       if (err) {
-        return cb(err);
+        return cb(err, '')
       }
-      cb(null, raw.toString('hex') + path.extname(file.originalname));
-    });
+      cb(null, raw.toString('hex') + path.extname(file.originalname))
+    })
   }
-});
+})
 
-function Upload(limits: multer.Options['limits'] = {}) {
-  return multer({
-    storage, limits
-  });
+function UploadMiddleware(limits: multer.Options['limits'] = {}): multer.Multer {
+  return multer({ storage, limits })
 }
 
-const defaultHandler = Upload()
-
-const upload = {
-  Upload, defaultHandler
-}
+const defaultHandler = UploadMiddleware()
+const upload = Object.freeze({ defaultHandler })
 
 export = upload

From 22ab9a6f8eebd388fbf746518c8225d691ec03a8 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 21 Aug 2024 15:22:24 -0600
Subject: [PATCH 069/183] refactor(service): users typescript/di: update todo
 comment

---
 service/src/routes/users.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/service/src/routes/users.js b/service/src/routes/users.js
index 82785e24a..e283ee19d 100644
--- a/service/src/routes/users.js
+++ b/service/src/routes/users.js
@@ -478,9 +478,8 @@ module.exports = function (app, security) {
     }
   );
 
-  // Update a specific user's password
-  // Need UPDATE_USER_PASSWORD to change a users password
-  // TODO this needs to be update to use the UPDATE_USER_PASSWORD permission when Android is updated to handle that permission
+  // TODO: this needs to be update to use the UPDATE_USER_PASSWORD permission when Android is updated to handle that permission
+  // TODO: updated: android is fine now.  create a migration to add UPDATE_USER_PASSWORD to the ADMIN_ROLE
   app.put(
     '/api/users/:userId/password',
     passport.authenticate('bearer'),

From bbbabffc32883c2767db2f3e98414969b5dd59b2 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 28 Aug 2024 09:05:04 -0600
Subject: [PATCH 070/183] refactor(service): users/auth: progress commit

---
 .../adapters.authentication.db.mongoose.ts    |  30 ++-
 service/src/authentication/devices.ts         |   4 +
 .../authentication/entities.authentication.ts |  51 +++-
 service/src/authentication/index.ts           | 232 +++++++++++++-----
 service/src/authentication/local.js           |   2 +-
 service/src/authentication/oauth.js           |   1 +
 6 files changed, 248 insertions(+), 72 deletions(-)
 create mode 100644 service/src/authentication/devices.ts

diff --git a/service/src/authentication/adapters.authentication.db.mongoose.ts b/service/src/authentication/adapters.authentication.db.mongoose.ts
index 4ab7ba73f..0e33445c7 100644
--- a/service/src/authentication/adapters.authentication.db.mongoose.ts
+++ b/service/src/authentication/adapters.authentication.db.mongoose.ts
@@ -2,25 +2,25 @@ import crypto from 'crypto'
 import mongoose, { Schema } from 'mongoose'
 import { UserDocumentExpanded } from '../adapters/users/adapters.users.db.mongoose'
 import { UserId } from '../entities/users/entities.users'
+import { Session, SessionRepository } from './entities.authentication'
 
 export interface SessionDocument {
   token: string
   expirationDate: Date
-  userId?: mongoose.Types.ObjectId | undefined
+  userId: mongoose.Types.ObjectId
   deviceId?: mongoose.Types.ObjectId | undefined
 }
 export type SessionDocumentExpanded = SessionDocument & {
   userId: UserDocumentExpanded
-  deviceId?: mongoose.Document
 }
 export type SessionModel = mongoose.Model<SessionDocument>
 
 const SessionSchema = new Schema<SessionDocument, SessionModel>(
   {
     token: { type: String, required: true },
+    expirationDate: { type: Date, required: true },
     userId: { type: Schema.Types.ObjectId, ref: 'User' },
     deviceId: { type: Schema.Types.ObjectId, ref: 'Device' },
-    expirationDate: { type: Date, required: true },
   },
   { versionKey: false }
 )
@@ -29,14 +29,6 @@ const SessionSchema = new Schema<SessionDocument, SessionModel>(
 SessionSchema.index({ token: 1 })
 SessionSchema.index({ expirationDate: 1 }, { expireAfterSeconds: 0 })
 
-export interface SessionRepository {
-  readonly model: SessionModel
-  createOrRefreshSession(userId: UserId, deviceId?: string): Promise<SessionDocumentExpanded>
-  removeSession(token: string): Promise<void>
-  removeSessionsForUser(userId: UserId): Promise<number>
-  removeSessionForDevice(deviceId: string): Promise<number>
-}
-
 const populateSessionUserRole: mongoose.PopulateOptions = {
   path: 'userId',
   populate: 'roleId'
@@ -46,7 +38,19 @@ export function createSessionRepository(conn: mongoose.Connection, collectionNam
   const model = conn.model('Token', SessionSchema, collectionName)
   return Object.freeze({
     model,
-    async createOrRefreshSession(userId: UserId, deviceId?: string): Promise<mongoose.HydratedDocument<SessionDocumentExpanded>> {
+    async findSessionByToken(token: string): Promise<Session | null> {
+      const doc = await model.findOne({ token }).lean()
+      if (!doc) {
+        return null
+      }
+      return {
+        token: doc.token,
+        expirationDate: doc.expirationDate,
+        user: doc.userId.toHexString(),
+        device: doc.deviceId?.toHexString(),
+      }
+    },
+    async createOrRefreshSession(userId: UserId, deviceId?: string): Promise<mongoose.HydratedDocument<Session>> {
       const seed = crypto.randomBytes(20)
       const token = crypto.createHash('sha256').update(seed).digest('hex')
       const query: any = { userId: new mongoose.Types.ObjectId(userId) }
@@ -68,7 +72,7 @@ export function createSessionRepository(conn: mongoose.Connection, collectionNam
       const { deletedCount } = await model.deleteMany({ userId: new mongoose.Types.ObjectId(userId) })
       return deletedCount
     },
-    async removeSessionForDevice(deviceId: string): Promise<number> {
+    async removeSessionsForDevice(deviceId: string): Promise<number> {
       const { deletedCount } = await model.deleteMany({ deviceId: new mongoose.Types.ObjectId(deviceId) })
       return deletedCount
     }
diff --git a/service/src/authentication/devices.ts b/service/src/authentication/devices.ts
new file mode 100644
index 000000000..b1a5a3aa3
--- /dev/null
+++ b/service/src/authentication/devices.ts
@@ -0,0 +1,4 @@
+
+export interface DeviceAuthenticationPolicyService {
+
+}
\ No newline at end of file
diff --git a/service/src/authentication/entities.authentication.ts b/service/src/authentication/entities.authentication.ts
index 370e9ecc4..6c211769f 100644
--- a/service/src/authentication/entities.authentication.ts
+++ b/service/src/authentication/entities.authentication.ts
@@ -1,4 +1,6 @@
 import { Device, DeviceId } from '../entities/devices/entities.devices'
+import { MageEventId } from '../entities/events/entities.events'
+import { TeamId } from '../entities/teams/entities.teams'
 import { UserExpanded, UserId } from '../entities/users/entities.users'
 
 export interface Session {
@@ -18,5 +20,52 @@ export interface SessionRepository {
   createOrRefreshSession(userId: UserId, deviceId?: string): Promise<Session>
   removeSession(token: string): Promise<void>
   removeSessionsForUser(userId: UserId): Promise<number>
-  removeSessionForDevice(deviceId: DeviceId): Promise<number>
+  removeSessionsForDevice(deviceId: DeviceId): Promise<number>
+}
+
+/**
+ * An authentication protocol defines the sequence of messages between a user, a service provider (Mage), and an
+ * identity provider (Google, Meta, Okta, Auth0, Microsoft, GitHub) necessary to securely inform the service provider
+ * that the user is valid.  Authentication protocols are OpenID Connect, OAuth, SAML, LDAP, and Mage's own local
+ * password authentication database.
+ */
+export interface AuthenticationProtocol {
+  name: string
+}
+
+/**
+ * An identity provider (IDP) is a service maintains user profiles and that Mage trusts to authenticate user
+ * credentials via a specific authentication protocol.  Mage delegates user authentication to identity providers.
+ * Within Mage, the identity provider implementation maps the provider's user profile/account attributes to a Mage
+ * user profile.
+ */
+export interface IdentityProvider {
+  name: string
+  title: string
+  protocol: AuthenticationProtocol
+  protocolSettings: Record<string, any>
+  enabled: boolean
+  lastUpdated: Date
+  enrollmentPolicy: EnrollmentPolicy
+  description?: string | null
+  textColor?: string | null
+  buttonColor?: string | null
+  icon?: Buffer | null
+}
+
+/**
+ * Enrollment policy defines rules and effects to apply when a new user establishes a Mage account.
+ */
+export interface EnrollmentPolicy {
+  assignToTeams: TeamId[]
+  assignToEvents: MageEventId[]
+  requireAccountApproval: boolean
+  requireDeviceApproval: boolean
+}
+
+
+
+export interface IdentityProviderRepository {
+  findById(): Promise<IdentityProvider | null>
+  findByName(name: string): Promise<IdentityProvider | null>
 }
\ No newline at end of file
diff --git a/service/src/authentication/index.ts b/service/src/authentication/index.ts
index 7335061b7..abec4966d 100644
--- a/service/src/authentication/index.ts
+++ b/service/src/authentication/index.ts
@@ -1,32 +1,152 @@
-const crypto = require('crypto')
-  , verification = require('./verification')
-  , api = require('../api/')
-  , config = require('../config.js')
-  , log = require('../logger')
-  , userTransformer = require('../transformers/user')
-  , authenticationApiAppender = require('../utilities/authenticationApiAppender')
-  , AuthenticationConfiguration = require('../models/authenticationconfiguration')
-  , SecurePropertyAppender = require('../security/utilities/secure-property-appender');
-
-const JWTService = verification.JWTService;
-const TokenAssertion = verification.TokenAssertion;
-
-class AuthenticationInitializer {
+import crypto from 'crypto'
+import { JWTService, Payload, TokenAssertion } from './verification'
+import express from 'express'
+import passport from 'passport'
+import provision, { ProvisionStatic } from '../provision'
+import { User, UserRepository } from '../entities/users/entities.users'
+import bearer from 'passport-http-bearer'
+import { UserDocument } from '../adapters/users/adapters.users.db.mongoose'
+import { SessionRepository } from './entities.authentication'
+const api = require('../api/')
+const config = require('../config.js')
+const log = require('../logger')
+const userTransformer = require('../transformers/user')
+const authenticationApiAppender = require('../utilities/authenticationApiAppender')
+const AuthenticationConfiguration = require('../models/authenticationconfiguration')
+const SecurePropertyAppender = require('../security/utilities/secure-property-appender');
+
+
+
+export async function initializeAuthenticationStack(
+  userRepo: UserRepository,
+  sessionRepo: SessionRepository,
+  verificationService: JWTService,
+  provisioning: provision.ProvisionStatic,
+  passport: passport.Authenticator,
+): Promise<express.Router> {
+  passport.serializeUser((user, done) => done(null, user.id))
+  passport.deserializeUser(async (id, done) => {
+    try {
+      const user = await userRepo.findById(String(id))
+      done(null, user)
+    }
+    catch (err) {
+      done(err)
+    }
+  })
+  registerAuthenticatedBearerTokenHandling(passport, sessionRepo, userRepo)
+  registerIdpAuthenticationVerification(passport, verificationService, userRepo)
+  const routes = express.Router()
+  registerTokenGenerationEndpointWithDeviceVerification(routes, passport)
+}
+
+const VerifyIdpAuthenticationToken = 'verifyIdpAuthenticationToken'
+
+function registerAuthenticatedBearerTokenHandling(passport: passport.Authenticator, sessionRepo: SessionRepository, userRepo: UserRepository): passport.Authenticator {
+  return passport.use(
+    /*
+    This is the default bearer token authentication, registered to the passport instance under the default `bearer`
+    name.
+    */
+    new bearer.Strategy(
+      { passReqToCallback: true },
+      async function (req: express.Request, token: string, done: (err: Error | null, user?: User, access?: bearer.IVerifyOptions) => any) {
+        try {
+          const session = await sessionRepo.findSessionByToken(token)
+          if (!session) {
+            console.warn('no session for token', token, req.method, req.url)
+            return done(null)
+          }
+          const user = await userRepo.findById(session.user)
+          if (!user) {
+            console.warn('no user for token', token, 'user id', session.user, req.method, req.url)
+            return done(null)
+          }
+          req.token = session.token
+          if (session.device) {
+            req.provisionedDeviceId = session.device
+          }
+          return done(null, user, { scope: 'all' });
+        }
+        catch (err) {
+          return done(err as Error)
+        }
+      }
+    )
+  )
+}
+
+/**
+ * Register a `BearerStrategy` that expects a JWT in the `Authorization` header that contains the
+ * {@link TokenAssertion.Authorized} claim.  The claim indicates the subject has authenticated with an IDP and can
+ * continue the sign-in process.  Decode and verify the JWT signature, retrieve the `User` for the JWT subject, and set
+ * `Request.user`.
+ */
+function registerIdpAuthenticationVerification(passport: passport.Authenticator, verificationService: JWTService, userRepo: UserRepository): passport.Authenticator {
+  passport.use(VerifyIdpAuthenticationToken, new bearer.Strategy(async function(token, done: (error: any, user?: User) => any) {
+    try {
+      const expectation: Payload = { assertion: TokenAssertion.Authorized, subject: null, expiration: null }
+      const payload = await verificationService.verifyToken(token, expectation)
+      const user = payload.subject ? await userRepo.findById(payload.subject) : null
+      if (user) {
+        return done(null, user)
+      }
+      done(new Error(`user id ${payload.subject} not found for transient token ${String(payload)}`))
+    }
+    catch (err) {
+      done(err)
+    }
+  }))
+  return passport
+}
+
+function registerDeviceVerificationAndTokenGenerationEndpoint(routes: express.Router, passport: passport.Authenticator, deviceProvisioning: ProvisionStatic, sessionRepo: SessionRepository) {
+  routes.post('/auth/token',
+    passport.authenticate(VerifyIdpAuthenticationToken),
+    async (req, res, next) => {
+      deviceProvisioning.check()
+      const options = {
+        userAgent: req.headers['user-agent'],
+        appVersion: req.body.appVersion
+      }
+      // TODO: users-next
+      new api.User().login(req.user, req.provisionedDevice, options, function (err, session) {
+        if (err) return next(err);
+
+        authenticationApiAppender.append(config.api).then(api => {
+          res.json({
+            token: session.token,
+            expirationDate: session.expirationDate,
+            user: userTransformer.transform(req.user, { path: req.getRoot() }),
+            device: req.provisionedDevice,
+            api: api
+          });
+        }).catch(err => {
+          next(err);
+        });
+      });
+
+      req.session = null;
+    }
+  );
+}
+
+function registerLocalAuthenticationProtocol(): void {
+
+}
+
+
+export class AuthenticationInitializer {
   static tokenService = new JWTService(crypto.randomBytes(64).toString('hex'), 'urn:mage');
-  static app;
-  static passport;
-  static provision;
+  static app: express.Application
+  static passport: passport.Authenticator;
+  static provision: provision.ProvisionStatic;
 
-  static initialize(app, passport, provision) {
+  static initialize(app: express.Application, passport: passport.Authenticator, provision: provision.ProvisionStatic): { passport: passport.Authenticator } {
     AuthenticationInitializer.app = app;
     AuthenticationInitializer.passport = passport;
     AuthenticationInitializer.provision = provision;
 
-    const BearerStrategy = require('passport-http-bearer').Strategy;
-    // TODO: users-next
-    const User = require('../models/user');
-    const Token = require('../models/token');
-
     passport.serializeUser(function (user, done) {
       done(null, user._id);
     });
@@ -38,26 +158,27 @@ class AuthenticationInitializer {
       });
     });
 
-    passport.use(new BearerStrategy({
-      passReqToCallback: true
-    },
-      function (req, token, done) {
-        Token.getToken(token, function (err, credentials) {
-          if (err) { return done(err); }
-
-          if (!credentials || !credentials.user) {
-            return done(null, false);
+    passport.use(
+      new bearer.Strategy(
+        { passReqToCallback: true },
+        async function (req: express.Request, token: string, done: (err: Error | null, user?: UserDocument, access?: bearer.IVerifyOptions) => any) {
+          try {
+            const session = await lookupSession(token)
+            if (!session || !session.user) {
+              return done(null)
+            }
+            req.token = session.token;
+            if (session.deviceId) {
+              req.provisionedDeviceId = session.deviceId.toHexString();
+            }
+            return done(null, session.user, { scope: 'all' });
           }
-
-          req.token = credentials.token;
-
-          if (credentials.token.deviceId) {
-            req.provisionedDeviceId = credentials.token.deviceId;
+          catch (err) {
+            return done(err as Error)
           }
-
-          return done(null, credentials.user, { scope: 'all' });
-        });
-      }));
+        }
+      )
+    )
 
     passport.use('authorization', new BearerStrategy(function (token, done) {
       const expectation = {
@@ -74,16 +195,17 @@ class AuthenticationInitializer {
         .catch(err => done(err));
     }));
 
-    function authorize(req, res, next) {
-      passport.authenticate('authorization', function (err, user, info = {}) {
-        if (!user) return res.status(401).send(info.message);
-
-        req.user = user;
-        next();
-      })(req, res, next);
+    function authorize(req: express.Request, res: express.Response, next: express.NextFunction): any {
+      passport.authenticate('authorization', function (err: Error, user: User, info: any = {}) {
+        if (!user) {
+          return res.status(401).send(info.message)
+        }
+        req.user = user
+        next()
+      })(req, res, next)
     }
 
-    function provisionDevice(req, res, next) {
+    function provisionDevice(req: express.Request, res: express.Response, next: express.NextFunction): any {
       provision.check(req.user.authentication.authenticationConfiguration.type, req.user.authentication.authenticationConfiguration.name)(req, res, next);
     }
 
@@ -96,13 +218,13 @@ class AuthenticationInitializer {
           appVersion: req.param('appVersion')
         };
         // TODO: users-next
-        new api.User().login(req.user, req.provisionedDevice, options, function (err, token) {
+        new api.User().login(req.user, req.provisionedDevice, options, function (err, session) {
           if (err) return next(err);
 
           authenticationApiAppender.append(config.api).then(api => {
             res.json({
-              token: token.token,
-              expirationDate: token.expirationDate,
+              token: session.token,
+              expirationDate: session.expirationDate,
               user: userTransformer.transform(req.user, { path: req.getRoot() }),
               device: req.provisionedDevice,
               api: api
@@ -134,10 +256,6 @@ class AuthenticationInitializer {
     require('./local').initialize();
     require('./anonymous').initialize();
 
-    return {
-      passport: passport
-    };
+    return { passport }
   }
 }
-
-module.exports = AuthenticationInitializer;
diff --git a/service/src/authentication/local.js b/service/src/authentication/local.js
index 5193d960f..0a70175fb 100644
--- a/service/src/authentication/local.js
+++ b/service/src/authentication/local.js
@@ -89,7 +89,7 @@ function initialize() {
       })(req, res, next);
     }
   );
-};
+}
 
 module.exports = {
   initialize
diff --git a/service/src/authentication/oauth.js b/service/src/authentication/oauth.js
index 63ce786fb..d0f3fec5a 100644
--- a/service/src/authentication/oauth.js
+++ b/service/src/authentication/oauth.js
@@ -74,6 +74,7 @@ function configure(strategy) {
       const profileId = profile[strategy.settings.profile.id];
 
       // TODO: users-next
+      // TODO: should be by strategy name, not strategy type
       User.getUserByAuthenticationStrategy(strategy.type, profileId, function (err, user) {
          if (err) return done(err);
 

From d26fe11cb925cd6b6a81f39e775b6ed5416b971d Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 28 Aug 2024 09:07:04 -0600
Subject: [PATCH 071/183] refactor(service)!: users/auth: remove deprecated
 devices/authorization routes from saml auth protocol

---
 service/src/authentication/saml.js | 151 ++++++++++++++---------------
 1 file changed, 75 insertions(+), 76 deletions(-)

diff --git a/service/src/authentication/saml.js b/service/src/authentication/saml.js
index 85ab11373..9479c16b2 100644
--- a/service/src/authentication/saml.js
+++ b/service/src/authentication/saml.js
@@ -2,12 +2,9 @@ const SamlStrategy = require('@node-saml/passport-saml').Strategy
   , log = require('winston')
   , User = require('../models/user')
   , Role = require('../models/role')
-  , Device = require('../models/device')
   , TokenAssertion = require('./verification').TokenAssertion
   , api = require('../api')
-  , userTransformer = require('../transformers/user')
   , AuthenticationInitializer = require('./index')
-  , authenticationApiAppender = require('../utilities/authenticationApiAppender');
 
 function configure(strategy) {
   log.info('Configuring ' + strategy.title + ' authentication');
@@ -221,19 +218,19 @@ function setDefaults(strategy) {
 function initialize(strategy) {
   const app = AuthenticationInitializer.app;
   const passport = AuthenticationInitializer.passport;
-  const provision = AuthenticationInitializer.provision;
+  // const provision = AuthenticationInitializer.provision;
 
   setDefaults(strategy);
   configure(strategy);
 
-  function parseLoginMetadata(req, res, next) {
-    req.loginOptions = {
-      userAgent: req.headers['user-agent'],
-      appVersion: req.param('appVersion')
-    };
+  // function parseLoginMetadata(req, res, next) {
+  //   req.loginOptions = {
+  //     userAgent: req.headers['user-agent'],
+  //     appVersion: req.param('appVersion')
+  //   };
 
-    next();
-  }
+  //   next();
+  // }
   app.get(
     '/auth/' + strategy.name + '/signin',
     function (req, res, next) {
@@ -252,79 +249,81 @@ function initialize(strategy) {
   // Create a new device
   // Any authenticated user can create a new device, the registered field
   // will be set to false.
-  app.post('/auth/' + strategy.name + '/devices',
-    function (req, res, next) {
-      if (req.user) {
-        next();
-      } else {
-        res.sendStatus(401);
-      }
-    },
-    function (req, res, next) {
-      const newDevice = {
-        uid: req.param('uid'),
-        name: req.param('name'),
-        registered: false,
-        description: req.param('description'),
-        userAgent: req.headers['user-agent'],
-        appVersion: req.param('appVersion'),
-        userId: req.user.id
-      };
+  // TODO: users-next: is this ok to remove now?
+  // app.post('/auth/' + strategy.name + '/devices',
+  //   function (req, res, next) {
+  //     if (req.user) {
+  //       next();
+  //     } else {
+  //       res.sendStatus(401);
+  //     }
+  //   },
+  //   function (req, res, next) {
+  //     const newDevice = {
+  //       uid: req.param('uid'),
+  //       name: req.param('name'),
+  //       registered: false,
+  //       description: req.param('description'),
+  //       userAgent: req.headers['user-agent'],
+  //       appVersion: req.param('appVersion'),
+  //       userId: req.user.id
+  //     };
 
-      Device.getDeviceByUid(newDevice.uid)
-        .then(device => {
-          if (device) {
-            // already exists, do not register
-            return res.json(device);
-          }
+  //     Device.getDeviceByUid(newDevice.uid)
+  //       .then(device => {
+  //         if (device) {
+  //           // already exists, do not register
+  //           return res.json(device);
+  //         }
 
-          Device.createDevice(newDevice)
-            .then(device => res.json(device))
-            .catch(err => next(err));
-        })
-        .catch(err => next(err));
-    }
-  );
+  //         Device.createDevice(newDevice)
+  //           .then(device => res.json(device))
+  //           .catch(err => next(err));
+  //       })
+  //       .catch(err => next(err));
+  //   }
+  // );
 
   // DEPRECATED session authorization, remove in next version.
-  app.post(
-    '/auth/' + strategy.name + '/authorize',
-    function (req, res, next) {
-      if (req.user) {
-        log.warn('session authorization is deprecated, please use jwt');
-        return next();
-      }
+  // TODO: users-next: is this ok to remove now? no other auth type has this
+  // app.post(
+  //   '/auth/' + strategy.name + '/authorize',
+  //   function (req, res, next) {
+  //     if (req.user) {
+  //       log.warn('session authorization is deprecated, please use jwt');
+  //       return next();
+  //     }
 
-      passport.authenticate('authorization', function (err, user, info = {}) {
-        if (!user) return res.status(401).send(info.message);
+  //     passport.authenticate('authorization', function (err, user, info = {}) {
+  //       if (!user) return res.status(401).send(info.message);
 
-        req.user = user;
-        next();
-      })(req, res, next);
-    },
-    provision.check(strategy.name),
-    parseLoginMetadata,
-    function (req, res, next) {
-      // TODO: users-next
-      new api.User().login(req.user, req.provisionedDevice, req.loginOptions, function (err, token) {
-        if (err) return next(err);
+  //       req.user = user;
+  //       next();
+  //     })(req, res, next);
+  //   },
+  //   provision.check(strategy.name),
+  //   parseLoginMetadata,
+  //   function (req, res, next) {
+  //     // TODO: users-next
+  //     new api.User().login(req.user, req.provisionedDevice, req.loginOptions, function (err, token) {
+  //       if (err) return next(err);
 
-        authenticationApiAppender.append(strategy.api).then(api => {
-          res.json({
-            token: token.token,
-            expirationDate: token.expirationDate,
-            user: userTransformer.transform(req.user, { path: req.getRoot() }),
-            device: req.provisionedDevice,
-            api: api
-          });
-        }).catch(err => {
-          next(err);
-        });
-      });
+  //       authenticationApiAppender.append(strategy.api).then(api => {
+  //         res.json({
+  //           token: token.token,
+  //           expirationDate: token.expirationDate,
+  //           user: userTransformer.transform(req.user, { path: req.getRoot() }),
+  //           device: req.provisionedDevice,
+  //           api: api
+  //         });
+  //       }).catch(err => {
+  //         next(err);
+  //       });
+  //     });
 
-      req.session = null;
-    }
-  );
+  //     req.session = null;
+  //   }
+  // );
 }
 
 module.exports = {

From 421d9d02c8563653bd6567975bc9e0fb6d8477ce Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 28 Aug 2024 09:19:32 -0600
Subject: [PATCH 072/183] refactor(service): users/auth: rename authentication
 folder to ingress to better reflect the broader purpose of enrollment,
 authentication, and identity management

---
 .../adapters.authentication.db.mongoose.ts                        | 0
 service/src/{authentication => ingress}/anonymous.js              | 0
 service/src/{authentication => ingress}/devices.ts                | 0
 .../src/{authentication => ingress}/entities.authentication.ts    | 0
 service/src/{authentication => ingress}/index.d.ts                | 0
 service/src/{authentication => ingress}/index.ts                  | 0
 service/src/{authentication => ingress}/ldap.js                   | 0
 service/src/{authentication => ingress}/local.js                  | 0
 service/src/{authentication => ingress}/oauth.js                  | 0
 service/src/{authentication => ingress}/openidconnect.js          | 0
 service/src/{authentication => ingress}/saml.js                   | 0
 service/src/{authentication => ingress}/verification.ts           | 0
 12 files changed, 0 insertions(+), 0 deletions(-)
 rename service/src/{authentication => ingress}/adapters.authentication.db.mongoose.ts (100%)
 rename service/src/{authentication => ingress}/anonymous.js (100%)
 rename service/src/{authentication => ingress}/devices.ts (100%)
 rename service/src/{authentication => ingress}/entities.authentication.ts (100%)
 rename service/src/{authentication => ingress}/index.d.ts (100%)
 rename service/src/{authentication => ingress}/index.ts (100%)
 rename service/src/{authentication => ingress}/ldap.js (100%)
 rename service/src/{authentication => ingress}/local.js (100%)
 rename service/src/{authentication => ingress}/oauth.js (100%)
 rename service/src/{authentication => ingress}/openidconnect.js (100%)
 rename service/src/{authentication => ingress}/saml.js (100%)
 rename service/src/{authentication => ingress}/verification.ts (100%)

diff --git a/service/src/authentication/adapters.authentication.db.mongoose.ts b/service/src/ingress/adapters.authentication.db.mongoose.ts
similarity index 100%
rename from service/src/authentication/adapters.authentication.db.mongoose.ts
rename to service/src/ingress/adapters.authentication.db.mongoose.ts
diff --git a/service/src/authentication/anonymous.js b/service/src/ingress/anonymous.js
similarity index 100%
rename from service/src/authentication/anonymous.js
rename to service/src/ingress/anonymous.js
diff --git a/service/src/authentication/devices.ts b/service/src/ingress/devices.ts
similarity index 100%
rename from service/src/authentication/devices.ts
rename to service/src/ingress/devices.ts
diff --git a/service/src/authentication/entities.authentication.ts b/service/src/ingress/entities.authentication.ts
similarity index 100%
rename from service/src/authentication/entities.authentication.ts
rename to service/src/ingress/entities.authentication.ts
diff --git a/service/src/authentication/index.d.ts b/service/src/ingress/index.d.ts
similarity index 100%
rename from service/src/authentication/index.d.ts
rename to service/src/ingress/index.d.ts
diff --git a/service/src/authentication/index.ts b/service/src/ingress/index.ts
similarity index 100%
rename from service/src/authentication/index.ts
rename to service/src/ingress/index.ts
diff --git a/service/src/authentication/ldap.js b/service/src/ingress/ldap.js
similarity index 100%
rename from service/src/authentication/ldap.js
rename to service/src/ingress/ldap.js
diff --git a/service/src/authentication/local.js b/service/src/ingress/local.js
similarity index 100%
rename from service/src/authentication/local.js
rename to service/src/ingress/local.js
diff --git a/service/src/authentication/oauth.js b/service/src/ingress/oauth.js
similarity index 100%
rename from service/src/authentication/oauth.js
rename to service/src/ingress/oauth.js
diff --git a/service/src/authentication/openidconnect.js b/service/src/ingress/openidconnect.js
similarity index 100%
rename from service/src/authentication/openidconnect.js
rename to service/src/ingress/openidconnect.js
diff --git a/service/src/authentication/saml.js b/service/src/ingress/saml.js
similarity index 100%
rename from service/src/authentication/saml.js
rename to service/src/ingress/saml.js
diff --git a/service/src/authentication/verification.ts b/service/src/ingress/verification.ts
similarity index 100%
rename from service/src/authentication/verification.ts
rename to service/src/ingress/verification.ts

From 5721387f7eb959eaf07308a0e98aea906284f694 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 28 Aug 2024 09:25:53 -0600
Subject: [PATCH 073/183] refactor(service): remove unsupported observation
 query fields `fields` and `geometry` from openapi doc

---
 service/src/docs/openapi.yaml | 43 ++++-------------------------------
 1 file changed, 4 insertions(+), 39 deletions(-)

diff --git a/service/src/docs/openapi.yaml b/service/src/docs/openapi.yaml
index e91abc3aa..f2912c75e 100644
--- a/service/src/docs/openapi.yaml
+++ b/service/src/docs/openapi.yaml
@@ -1647,16 +1647,6 @@ paths:
         user must have `READ_OBSERVATION_ALL` permission, or have
         `READ_OBSERVATION_EVENT` permission and an ACL entry on the event with
         `read` permission.
-      parameters: &obsQueryParams
-        - $ref: '#/components/parameters/observationQuery.fields'
-        - $ref: '#/components/parameters/observationQuery.startDate'
-        - $ref: '#/components/parameters/observationQuery.endDate'
-        - $ref: '#/components/parameters/observationQuery.observationStartDate'
-        - $ref: '#/components/parameters/observationQuery.observationEndDate'
-        - $ref: '#/components/parameters/observationQuery.bbox'
-        - $ref: '#/components/parameters/observationQuery.geometry'
-        - $ref: '#/components/parameters/observationQuery.states'
-        - $ref: '#/components/parameters/observationQuery.sort'
       responses:
         200:
           description: observation response
@@ -1703,13 +1693,11 @@ paths:
       # the update to js-yaml 4.1.x through openapi-enforcer, yaml anchors seem
       # to be broken
       parameters:
-        - $ref: '#/components/parameters/observationQuery.fields'
         - $ref: '#/components/parameters/observationQuery.startDate'
         - $ref: '#/components/parameters/observationQuery.endDate'
         - $ref: '#/components/parameters/observationQuery.observationStartDate'
         - $ref: '#/components/parameters/observationQuery.observationEndDate'
         - $ref: '#/components/parameters/observationQuery.bbox'
-        - $ref: '#/components/parameters/observationQuery.geometry'
         - $ref: '#/components/parameters/observationQuery.states'
         - $ref: '#/components/parameters/observationQuery.sort'
       responses:
@@ -4372,16 +4360,6 @@ components:
       description: The ID of the target attachment document
       required: true
       schema: { $ref: '#/components/schemas/Attachment/properties/id' }
-    observationQuery.fields:
-      in: query
-      name: fields
-      description: >
-        The form fields to project in the result observation documents (JSON)
-      explode: false
-      schema:
-        type: array
-        items:
-          type: string
     observationQuery.startDate:
       in: query
       name: startDate
@@ -4389,8 +4367,7 @@ components:
         type: string
         format: date-time
       description: >
-        The low end of the range for the observations' `lastModified`
-        property
+        The low end of the range for the observations' `lastModified` property
     observationQuery.endDate:
       in: query
       name: endDate
@@ -4398,8 +4375,7 @@ components:
         type: string
         format: date-time
       description: >
-        The high end of the range for the observations' `lastModified`
-        property
+        The high end of the range for the observations' `lastModified` property
     observationQuery.observationStartDate:
       in: query
       name: observationStartDate
@@ -4407,8 +4383,7 @@ components:
         type: string
         format: date-time
       description: >
-        The low end of the range for the observations' `timestamp`
-        property
+        The low end of the range for the observations' `timestamp` property
     observationQuery.observationEndDate:
       in: query
       name: observationEndDate
@@ -4416,8 +4391,7 @@ components:
         type: string
         format: date-time
       description: >
-        The low end of the range for the observations' `lastModified`
-        property
+        The high end of the range for the observations' `timestamp` property
     observationQuery.bbox:
       in: query
       name: bbox
@@ -4427,15 +4401,6 @@ components:
       explode: false
       schema:
         $ref: 'geojson.yaml#/definitions/boundingBox'
-    observationQuery.geometry:
-      in: query
-      name: geometry
-      description: >
-        A URL-encoded, stringified JSON object that is a GeoJSON geometry
-        as defined in geojson.yaml#/definitions/geometryObject
-      schema:
-        type: string
-        format: json
     observationQuery.states:
       in: query
       name: states

From dd0f3337420a324b62ca50e184af88afa318e8e3 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 28 Aug 2024 14:45:31 -0600
Subject: [PATCH 074/183] refactor(service): users/auth: return deleted session
 from repository remove method

---
 service/src/ingress/entities.authentication.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/ingress/entities.authentication.ts b/service/src/ingress/entities.authentication.ts
index 6c211769f..4e63c8365 100644
--- a/service/src/ingress/entities.authentication.ts
+++ b/service/src/ingress/entities.authentication.ts
@@ -18,7 +18,7 @@ export type SessionExpanded = Omit<Session, 'user' | 'device'> & {
 export interface SessionRepository {
   findSessionByToken(token: string): Promise<Session | null>
   createOrRefreshSession(userId: UserId, deviceId?: string): Promise<Session>
-  removeSession(token: string): Promise<void>
+  removeSession(token: string): Promise<Session | null>
   removeSessionsForUser(userId: UserId): Promise<number>
   removeSessionsForDevice(deviceId: DeviceId): Promise<number>
 }

From 27a8c80e70fb8e475814f4d2d89a9284c5d0939e Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 28 Aug 2024 15:20:12 -0600
Subject: [PATCH 075/183] refactor(service): users/auth: improve user entity
 docs and add user repository methods

---
 service/src/entities/users/entities.users.ts | 65 ++++++++++++++------
 1 file changed, 45 insertions(+), 20 deletions(-)

diff --git a/service/src/entities/users/entities.users.ts b/service/src/entities/users/entities.users.ts
index 9e90eaa24..a8bea7433 100644
--- a/service/src/entities/users/entities.users.ts
+++ b/service/src/entities/users/entities.users.ts
@@ -1,6 +1,5 @@
 import { PagingParameters, PageOf } from '../entities.global'
 import { Role } from '../authorization/entities.authorization'
-import { Authentication } from '../authentication/entities.authentication'
 import { MageEventId } from '../events/entities.events'
 
 export type UserId = string
@@ -9,30 +8,35 @@ export interface User {
   id: UserId
   username: string
   displayName: string
+  /**
+   * Active indicates whether an admin approved the user account.  This flag only ever changes value one time.
+   */
   active: boolean
+  /**
+   * The enabled flag indicates whether a user can access Mage and preform any operations.  An administrator can
+   * disable a user account at any time to block the user's access.
+   */
   enabled: boolean
   createdAt: Date
   lastUpdated: Date
+  roleId: string
   email?: string
   phones: Phone[]
-  roleId: string
-  authenticationId: string
+  /**
+   * A user's avatar is the profile picture that represents the user in list
+   * views and such.
+   * TODO: make this nullable rather than an empty object. that is a symptom of the mongoose schema. make sure a null value does not break clients
+   */
   avatar: Avatar
-  icon: UserIcon
   /**
-   * TODO: this could move to another entity like `UserExperience` or
-   * `UserSettings` to eliminate the cyclic reference between the user and
-   * event modules.
+   * A user's icon is to indicate the user's location on a map display.
    */
+  icon: UserIcon
   recentEventIds: MageEventId[]
-  // TODO: the rest of the properties
 }
 
-export type UserExpanded = Omit<User, 'roleId' | 'authenticationId'>
-  & {
-    role: Role
-    authentication: Authentication
-  }
+export type UserExpanded = Omit<User, 'roleId'>
+  & { role: Role }
 
 export interface Phone {
   type: string,
@@ -40,12 +44,17 @@ export interface Phone {
 }
 
 /**
- * A user's icon is what appears on the map to mark the user's location.
+ * TODO: There is not much value to retaining the `type`, `text`, and `color` attributes.  Only the web app's user
+ * admin screen uses these to set default form values, but the web app always generates a raster png from those values
+ * anyway.
  */
 export interface UserIcon {
+  /**
+   * Type defaults to {@link UserIconType.None} via database layer.
+   */
   type: UserIconType
-  text: string
-  color: string
+  text?: string
+  color?: string
   contentType?: string
   size?: number
   relativePath?: string
@@ -57,10 +66,6 @@ export enum UserIconType {
   Create = 'create',
 }
 
-/**
- * A user's avatar is the profile picture that represents the user in list
- * views and such.
- */
 export interface Avatar {
   contentType?: string,
   size?: number,
@@ -68,9 +73,18 @@ export interface Avatar {
 }
 
 export interface UserRepository {
+  create(userAttrs: Omit<User, 'id' | 'icon' | 'avatar'>): Promise<User | UserRepositoryError>
+  /**
+   * Return `null` if the specified user ID does not exist.
+   */
+  update(userAttrs: Partial<User> & Pick<User, 'id'>): Promise<User | null | UserRepositoryError>
   findById(id: UserId): Promise<User | null>
   findAllByIds(ids: UserId[]): Promise<{ [id: string]: User | null }>
   find<MappedResult>(which?: UserFindParameters, mapping?: (user: User) => MappedResult): Promise<PageOf<MappedResult>>
+  saveMapIcon(userId: UserId, icon: UserIcon, content: NodeJS.ReadableStream | Buffer): Promise<User | UserRepositoryError>
+  saveAvatar(avatar: Avatar, content: NodeJS.ReadableStream | Buffer): Promise<User | UserRepositoryError>
+  deleteMapIcon(userId: UserId): Promise<User | UserRepositoryError>
+  deleteAvatar(userId: UserId): Promise<User | UserRepositoryError>
 }
 
 export interface UserFindParameters extends PagingParameters {
@@ -81,3 +95,14 @@ export interface UserFindParameters extends PagingParameters {
   active?: boolean | undefined
   enabled?: boolean | undefined
 }
+
+export class UserRepositoryError extends Error {
+  constructor(public errorCode: UserRepositoryErrorCode, message?: string) {
+    super(message)
+  }
+}
+
+export enum UserRepositoryErrorCode {
+  DuplicateUserName = 'DuplicateUserName',
+  StorageError = 'StorageError',
+}
\ No newline at end of file

From 0330bcf92e1d1279207a2bfd9bd36fcebff14bb1 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 5 Sep 2024 08:48:44 -0600
Subject: [PATCH 076/183] refactor(service): users/auth: some todo notes

---
 service/src/ingress/entities.authentication.ts | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/service/src/ingress/entities.authentication.ts b/service/src/ingress/entities.authentication.ts
index 4e63c8365..0ee5bbc29 100644
--- a/service/src/ingress/entities.authentication.ts
+++ b/service/src/ingress/entities.authentication.ts
@@ -57,15 +57,16 @@ export interface IdentityProvider {
  * Enrollment policy defines rules and effects to apply when a new user establishes a Mage account.
  */
 export interface EnrollmentPolicy {
+  // TODO: configurable role assignment
+  // assignRole: string
   assignToTeams: TeamId[]
   assignToEvents: MageEventId[]
   requireAccountApproval: boolean
+  // TODO: move to different policy?
   requireDeviceApproval: boolean
 }
 
-
-
 export interface IdentityProviderRepository {
   findById(): Promise<IdentityProvider | null>
   findByName(name: string): Promise<IdentityProvider | null>
-}
\ No newline at end of file
+}

From 8b0624c0624e6e9633a8f09e0afe4733bb53c6a6 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 5 Sep 2024 14:12:31 -0600
Subject: [PATCH 077/183] refactor(service): users/auth: remove deprecated
 routes from saml auth handler

---
 service/src/ingress/saml.js | 80 -------------------------------------
 1 file changed, 80 deletions(-)

diff --git a/service/src/ingress/saml.js b/service/src/ingress/saml.js
index 9479c16b2..2376bd1f8 100644
--- a/service/src/ingress/saml.js
+++ b/service/src/ingress/saml.js
@@ -244,86 +244,6 @@ function initialize(strategy) {
       })(req, res, next);
     }
   );
-
-  // DEPRECATED retain old routes as deprecated until next major version.
-  // Create a new device
-  // Any authenticated user can create a new device, the registered field
-  // will be set to false.
-  // TODO: users-next: is this ok to remove now?
-  // app.post('/auth/' + strategy.name + '/devices',
-  //   function (req, res, next) {
-  //     if (req.user) {
-  //       next();
-  //     } else {
-  //       res.sendStatus(401);
-  //     }
-  //   },
-  //   function (req, res, next) {
-  //     const newDevice = {
-  //       uid: req.param('uid'),
-  //       name: req.param('name'),
-  //       registered: false,
-  //       description: req.param('description'),
-  //       userAgent: req.headers['user-agent'],
-  //       appVersion: req.param('appVersion'),
-  //       userId: req.user.id
-  //     };
-
-  //     Device.getDeviceByUid(newDevice.uid)
-  //       .then(device => {
-  //         if (device) {
-  //           // already exists, do not register
-  //           return res.json(device);
-  //         }
-
-  //         Device.createDevice(newDevice)
-  //           .then(device => res.json(device))
-  //           .catch(err => next(err));
-  //       })
-  //       .catch(err => next(err));
-  //   }
-  // );
-
-  // DEPRECATED session authorization, remove in next version.
-  // TODO: users-next: is this ok to remove now? no other auth type has this
-  // app.post(
-  //   '/auth/' + strategy.name + '/authorize',
-  //   function (req, res, next) {
-  //     if (req.user) {
-  //       log.warn('session authorization is deprecated, please use jwt');
-  //       return next();
-  //     }
-
-  //     passport.authenticate('authorization', function (err, user, info = {}) {
-  //       if (!user) return res.status(401).send(info.message);
-
-  //       req.user = user;
-  //       next();
-  //     })(req, res, next);
-  //   },
-  //   provision.check(strategy.name),
-  //   parseLoginMetadata,
-  //   function (req, res, next) {
-  //     // TODO: users-next
-  //     new api.User().login(req.user, req.provisionedDevice, req.loginOptions, function (err, token) {
-  //       if (err) return next(err);
-
-  //       authenticationApiAppender.append(strategy.api).then(api => {
-  //         res.json({
-  //           token: token.token,
-  //           expirationDate: token.expirationDate,
-  //           user: userTransformer.transform(req.user, { path: req.getRoot() }),
-  //           device: req.provisionedDevice,
-  //           api: api
-  //         });
-  //       }).catch(err => {
-  //         next(err);
-  //       });
-  //     });
-
-  //     req.session = null;
-  //   }
-  // );
 }
 
 module.exports = {

From 085dff23c2409c39e7e72a1999efd6105b496877 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 5 Sep 2024 16:40:28 -0600
Subject: [PATCH 078/183] refactor(service): users/auth: rename protocol js
 module files to typescript files

---
 .../src/ingress/{anonymous.js => ingress.protocol.anonymous.ts}   | 0
 service/src/ingress/{ldap.js => ingress.protocol.ldap.ts}         | 0
 service/src/ingress/{local.js => ingress.protocol.local.ts}       | 0
 service/src/ingress/{oauth.js => ingress.protocol.oauth.ts}       | 0
 .../src/ingress/{openidconnect.js => ingress.protocol.oidc.ts}    | 0
 service/src/ingress/{saml.js => ingress.protocol.saml.ts}         | 0
 6 files changed, 0 insertions(+), 0 deletions(-)
 rename service/src/ingress/{anonymous.js => ingress.protocol.anonymous.ts} (100%)
 rename service/src/ingress/{ldap.js => ingress.protocol.ldap.ts} (100%)
 rename service/src/ingress/{local.js => ingress.protocol.local.ts} (100%)
 rename service/src/ingress/{oauth.js => ingress.protocol.oauth.ts} (100%)
 rename service/src/ingress/{openidconnect.js => ingress.protocol.oidc.ts} (100%)
 rename service/src/ingress/{saml.js => ingress.protocol.saml.ts} (100%)

diff --git a/service/src/ingress/anonymous.js b/service/src/ingress/ingress.protocol.anonymous.ts
similarity index 100%
rename from service/src/ingress/anonymous.js
rename to service/src/ingress/ingress.protocol.anonymous.ts
diff --git a/service/src/ingress/ldap.js b/service/src/ingress/ingress.protocol.ldap.ts
similarity index 100%
rename from service/src/ingress/ldap.js
rename to service/src/ingress/ingress.protocol.ldap.ts
diff --git a/service/src/ingress/local.js b/service/src/ingress/ingress.protocol.local.ts
similarity index 100%
rename from service/src/ingress/local.js
rename to service/src/ingress/ingress.protocol.local.ts
diff --git a/service/src/ingress/oauth.js b/service/src/ingress/ingress.protocol.oauth.ts
similarity index 100%
rename from service/src/ingress/oauth.js
rename to service/src/ingress/ingress.protocol.oauth.ts
diff --git a/service/src/ingress/openidconnect.js b/service/src/ingress/ingress.protocol.oidc.ts
similarity index 100%
rename from service/src/ingress/openidconnect.js
rename to service/src/ingress/ingress.protocol.oidc.ts
diff --git a/service/src/ingress/saml.js b/service/src/ingress/ingress.protocol.saml.ts
similarity index 100%
rename from service/src/ingress/saml.js
rename to service/src/ingress/ingress.protocol.saml.ts

From 313475fc2b210a6b72c2baa75fcefb67f81f283b Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 5 Sep 2024 22:19:44 -0600
Subject: [PATCH 079/183] refactor(service): users/auth: add passport-local
 types dependency

---
 service/npm-shrinkwrap.json | 12 ++++++++++++
 service/package.json        |  1 +
 2 files changed, 13 insertions(+)

diff --git a/service/npm-shrinkwrap.json b/service/npm-shrinkwrap.json
index 1b8505970..effece5e4 100644
--- a/service/npm-shrinkwrap.json
+++ b/service/npm-shrinkwrap.json
@@ -90,6 +90,7 @@
         "@types/node-fetch": "^2.5.4",
         "@types/passport": "^1.0.3",
         "@types/passport-http-bearer": "^1.0.41",
+        "@types/passport-local": "^1.0.38",
         "@types/sinon": "^9.0.4",
         "@types/sinon-chai": "^3.2.4",
         "@types/superagent": "^8.1.3",
@@ -2738,6 +2739,17 @@
         "@types/passport": "*"
       }
     },
+    "node_modules/@types/passport-local": {
+      "version": "1.0.38",
+      "resolved": "https://registry.npmjs.org/@types/passport-local/-/passport-local-1.0.38.tgz",
+      "integrity": "sha512-nsrW4A963lYE7lNTv9cr5WmiUD1ibYJvWrpE13oxApFsRt77b0RdtZvKbCdNIY4v/QZ6TRQWaDDEwV1kCTmcXg==",
+      "dev": true,
+      "dependencies": {
+        "@types/express": "*",
+        "@types/passport": "*",
+        "@types/passport-strategy": "*"
+      }
+    },
     "node_modules/@types/passport-strategy": {
       "version": "0.2.38",
       "resolved": "https://registry.npmjs.org/@types/passport-strategy/-/passport-strategy-0.2.38.tgz",
diff --git a/service/package.json b/service/package.json
index 94eed6571..b6123637b 100644
--- a/service/package.json
+++ b/service/package.json
@@ -107,6 +107,7 @@
     "@types/node-fetch": "^2.5.4",
     "@types/passport": "^1.0.3",
     "@types/passport-http-bearer": "^1.0.41",
+    "@types/passport-local": "^1.0.38",
     "@types/sinon": "^9.0.4",
     "@types/sinon-chai": "^3.2.4",
     "@types/superagent": "^8.1.3",

From b0a660a9c7183d3f447cc11a096160a7faba506c Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 5 Sep 2024 22:55:06 -0600
Subject: [PATCH 080/183] refactor(service): users/auth: add username lookup to
 user repository

---
 service/src/entities/users/entities.users.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/service/src/entities/users/entities.users.ts b/service/src/entities/users/entities.users.ts
index a8bea7433..b8080852e 100644
--- a/service/src/entities/users/entities.users.ts
+++ b/service/src/entities/users/entities.users.ts
@@ -79,6 +79,7 @@ export interface UserRepository {
    */
   update(userAttrs: Partial<User> & Pick<User, 'id'>): Promise<User | null | UserRepositoryError>
   findById(id: UserId): Promise<User | null>
+  findByUsername(username: string): Promise<User | null>
   findAllByIds(ids: UserId[]): Promise<{ [id: string]: User | null }>
   find<MappedResult>(which?: UserFindParameters, mapping?: (user: User) => MappedResult): Promise<PageOf<MappedResult>>
   saveMapIcon(userId: UserId, icon: UserIcon, content: NodeJS.ReadableStream | Buffer): Promise<User | UserRepositoryError>

From 9759fcd41b79816c757a9b53b9716e32504e7ec7 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 6 Sep 2024 18:04:10 -0600
Subject: [PATCH 081/183] refactor(service): users/auth: add doc comment to
 base mongoose repository

---
 service/src/adapters/base/adapters.base.db.mongoose.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/service/src/adapters/base/adapters.base.db.mongoose.ts b/service/src/adapters/base/adapters.base.db.mongoose.ts
index 2111bff10..f9ac63082 100644
--- a/service/src/adapters/base/adapters.base.db.mongoose.ts
+++ b/service/src/adapters/base/adapters.base.db.mongoose.ts
@@ -101,6 +101,12 @@ export class BaseMongooseRepository<DocType extends mongoose.AnyObject, Model ex
     return { ...notFound, ...found }
   }
 
+  /**
+   * Use the {@link BaseMongooseRepository.documentStubForEntity entity mapping} to convert the given entity to a
+   * MongoDB document, and use that document to {@link mongoose.Document.set() set} the entries on the existing
+   * document.  To remove an entry from the persisted document, set the key's value to `undefined` in the given input
+   * document.  For example, `repo.update({ id: 'asdf0987', deleteThisKey: undefined })`.
+   */
   async update(attrs: Partial<Entity> & EntityReference): Promise<Entity | null> {
     let doc = (await this.model.findById(attrs.id))
     if (!doc) {

From f67dd4f3c332c3d6e08967b47e8cf74450bba339 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 9 Sep 2024 13:28:17 -0600
Subject: [PATCH 082/183] refactor(service): users/auth: wip: oauth2 typescript
 transition

---
 service/src/ingress/ingress.protocol.oauth.ts | 66 +++++++++++--------
 1 file changed, 37 insertions(+), 29 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.oauth.ts b/service/src/ingress/ingress.protocol.oauth.ts
index d0f3fec5a..6a9635273 100644
--- a/service/src/ingress/ingress.protocol.oauth.ts
+++ b/service/src/ingress/ingress.protocol.oauth.ts
@@ -1,42 +1,50 @@
 'use strict';
 
-const OAuth2Strategy = require('passport-oauth2').Strategy
-   , TokenAssertion = require('./verification').TokenAssertion
-   , base64 = require('base-64')
-   , api = require('../api')
-   , log = require('../logger')
-   , User = require('../models/user')
-   , Role = require('../models/role')
-   , { app, passport, tokenService } = require('./index');
+import { InternalOAuthError, Strategy as OAuth2Strategy, StrategyOptions as OAuth2Options, VerifyFunction } from 'passport-oauth2'
+import { TokenAssertion, JWTService } from './verification'
+import base64 from 'base-64'
+const api = require('../api')
+const log = require('../logger')
+const User = require('../models/user')
+const Role = require('../models/role')
+const { app, passport, tokenService } = require('./index');
+
+interface MageOAuth2Options extends OAuth2Options {
+   profileURL: string
+}
 
 class OAuth2ProfileStrategy extends OAuth2Strategy {
-   constructor(options, verify) {
-      super(options, verify);
 
-      if (!options.profileURL) { throw new TypeError('OAuth2Strategy requires a profileURL option'); }
-      this._profileURL = options.profileURL;
+   private _profileURL: string
 
-      this._oauth2.useAuthorizationHeaderforGET(true);
+   constructor(options: MageOAuth2Options, verify: VerifyFunction) {
+      super(options, verify)
+      if (!options.profileURL) {
+         throw new TypeError('OAuth2: missing profileURL')
+      }
+      this._profileURL = options.profileURL
+      this._oauth2.useAuthorizationHeaderforGET(true)
    }
 
-   userProfile(accessToken, done) {
-      this._oauth2.get(this._profileURL, accessToken, function (err, body) {
-         if (err) { return done(new InternalOAuthError('Failed to fetch user profile', err)); }
-
+   async userProfile(accessToken: string, done: (err: unknown, profile?: any) => void): Promise<void> {
+      this._oauth2.get(this._profileURL, accessToken, (err, body) => {
+         if (err) {
+            return done(new InternalOAuthError('error fetching oauth2 user profile', err))
+         }
          try {
-            const json = JSON.parse(body);
-
-            const profile = {};
-            profile.provider = 'oauth2';
-            profile.raw = body;
-            profile.json = json;
-
-            done(null, profile);
-         } catch (e) {
-            log.warn('Error parsing oauth profile', e);
-            done(e);
+            const parsedBody = JSON.parse(body as string)
+            const profile = {
+               provider: 'oauth2',
+               json: parsedBody,
+               raw: body,
+            }
+            done(null, profile)
          }
-      });
+         catch (err) {
+            log.error('error parsing oauth profile', err)
+            done(err)
+         }
+      })
    }
 }
 

From 37e8f7729b50a81e4e9e3ee8e7c7e32cff2e7326 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 9 Sep 2024 13:30:04 -0600
Subject: [PATCH 083/183] refactor(service): users/auth: oauth protocol white
 space

---
 service/src/ingress/ingress.protocol.oauth.ts | 398 +++++++++---------
 1 file changed, 199 insertions(+), 199 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.oauth.ts b/service/src/ingress/ingress.protocol.oauth.ts
index 6a9635273..79d980839 100644
--- a/service/src/ingress/ingress.protocol.oauth.ts
+++ b/service/src/ingress/ingress.protocol.oauth.ts
@@ -10,224 +10,224 @@ const Role = require('../models/role')
 const { app, passport, tokenService } = require('./index');
 
 interface MageOAuth2Options extends OAuth2Options {
-   profileURL: string
+  profileURL: string
 }
 
 class OAuth2ProfileStrategy extends OAuth2Strategy {
 
-   private _profileURL: string
-
-   constructor(options: MageOAuth2Options, verify: VerifyFunction) {
-      super(options, verify)
-      if (!options.profileURL) {
-         throw new TypeError('OAuth2: missing profileURL')
+  private _profileURL: string
+
+  constructor(options: MageOAuth2Options, verify: VerifyFunction) {
+    super(options, verify)
+    if (!options.profileURL) {
+      throw new TypeError('OAuth2: missing profileURL')
+    }
+    this._profileURL = options.profileURL
+    this._oauth2.useAuthorizationHeaderforGET(true)
+  }
+
+  async userProfile(accessToken: string, done: (err: unknown, profile?: any) => void): Promise<void> {
+    this._oauth2.get(this._profileURL, accessToken, (err, body) => {
+      if (err) {
+        return done(new InternalOAuthError('error fetching oauth2 user profile', err))
       }
-      this._profileURL = options.profileURL
-      this._oauth2.useAuthorizationHeaderforGET(true)
-   }
-
-   async userProfile(accessToken: string, done: (err: unknown, profile?: any) => void): Promise<void> {
-      this._oauth2.get(this._profileURL, accessToken, (err, body) => {
-         if (err) {
-            return done(new InternalOAuthError('error fetching oauth2 user profile', err))
-         }
-         try {
-            const parsedBody = JSON.parse(body as string)
-            const profile = {
-               provider: 'oauth2',
-               json: parsedBody,
-               raw: body,
-            }
-            done(null, profile)
-         }
-         catch (err) {
-            log.error('error parsing oauth profile', err)
-            done(err)
-         }
-      })
-   }
+      try {
+        const parsedBody = JSON.parse(body as string)
+        const profile = {
+          provider: 'oauth2',
+          json: parsedBody,
+          raw: body,
+        }
+        done(null, profile)
+      }
+      catch (err) {
+        log.error('error parsing oauth profile', err)
+        done(err)
+      }
+    })
+  }
 }
 
 function configure(strategy) {
-   log.info('Configuring ' + strategy.title + ' authentication');
-
-   let customHeaders = null;
-
-   if (strategy.settings.headers) {
-      customHeaders = {};
-      if (strategy.settings.headers.basic) {
-         customHeaders['Authorization'] = `Basic ${base64.encode(`${strategy.settings.clientID}:${strategy.settings.clientSecret}`)}`;
-      }
-   }
-
-   passport.use(strategy.name, new OAuth2ProfileStrategy({
-      clientID: strategy.settings.clientID,
-      clientSecret: strategy.settings.clientSecret,
-      callbackURL: `/auth/${strategy.name}/callback`,
-      authorizationURL: strategy.settings.authorizationURL,
-      tokenURL: strategy.settings.tokenURL,
-      profileURL: strategy.settings.profileURL,
-      customHeaders: customHeaders,
-      scope: strategy.settings.scope,
-      pkce: strategy.settings.pkce,
-      store: true
-   }, function (accessToken, refreshToken, profileResponse, done) {
-      const profile = profileResponse.json;
-
-      if (!profile[strategy.settings.profile.id]) {
-         log.warn("JSON: " + JSON.stringify(profile) + " RAW: " + profileResponse.raw);
-         return done(`OAuth2 user profile does not contain id property named ${strategy.settings.profile.id}`);
+  log.info('Configuring ' + strategy.title + ' authentication');
+
+  let customHeaders = null;
+
+  if (strategy.settings.headers) {
+    customHeaders = {};
+    if (strategy.settings.headers.basic) {
+      customHeaders['Authorization'] = `Basic ${base64.encode(`${strategy.settings.clientID}:${strategy.settings.clientSecret}`)}`;
+    }
+  }
+
+  passport.use(strategy.name, new OAuth2ProfileStrategy({
+    clientID: strategy.settings.clientID,
+    clientSecret: strategy.settings.clientSecret,
+    callbackURL: `/auth/${strategy.name}/callback`,
+    authorizationURL: strategy.settings.authorizationURL,
+    tokenURL: strategy.settings.tokenURL,
+    profileURL: strategy.settings.profileURL,
+    customHeaders: customHeaders,
+    scope: strategy.settings.scope,
+    pkce: strategy.settings.pkce,
+    store: true
+  }, function (accessToken, refreshToken, profileResponse, done) {
+    const profile = profileResponse.json;
+
+    if (!profile[strategy.settings.profile.id]) {
+      log.warn("JSON: " + JSON.stringify(profile) + " RAW: " + profileResponse.raw);
+      return done(`OAuth2 user profile does not contain id property named ${strategy.settings.profile.id}`);
+    }
+
+    const profileId = profile[strategy.settings.profile.id];
+
+    // TODO: users-next
+    // TODO: should be by strategy name, not strategy type
+    User.getUserByAuthenticationStrategy(strategy.type, profileId, function (err, user) {
+      if (err) return done(err);
+
+      if (!user) {
+        // Create an account for the user
+        Role.getRole('USER_ROLE', function (err, role) {
+          if (err) return done(err);
+
+          let email = null;
+          if (profile[strategy.settings.profile.email]) {
+            if (Array.isArray(profile[strategy.settings.profile.email])) {
+              email = profile[strategy.settings.profile.email].find(email => {
+                email.verified === true
+              });
+            } else {
+              email = profile[strategy.settings.profile.email];
+            }
+          } else {
+            log.warn(`OAuth2 user profile does not contain email property named ${strategy.settings.profile.email}`);
+            log.debug(JSON.stringify(profile));
+          }
+
+          const user = {
+            username: profileId,
+            displayName: profile[strategy.settings.profile.displayName] || profileId,
+            email: email,
+            active: false,
+            roleId: role._id,
+            authentication: {
+              type: strategy.type,
+              id: profileId,
+              authenticationConfiguration: {
+                name: strategy.name
+              }
+            }
+          };
+          // TODO: users-next
+          new api.User().create(user).then(newUser => {
+            if (!newUser.authentication.authenticationConfiguration.enabled) {
+              log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
+              return done(null, false, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
+            }
+            return done(null, newUser);
+          }).catch(err => done(err));
+        });
+      } else if (!user.active) {
+        return done(null, user, { message: "User is not approved, please contact your MAGE administrator to approve your account." });
+      } else if (!user.authentication.authenticationConfiguration.enabled) {
+        log.warn(user.authentication.authenticationConfiguration.title + " authentication is not enabled");
+        return done(null, user, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
+      } else {
+        return done(null, user);
       }
-
-      const profileId = profile[strategy.settings.profile.id];
-
-      // TODO: users-next
-      // TODO: should be by strategy name, not strategy type
-      User.getUserByAuthenticationStrategy(strategy.type, profileId, function (err, user) {
-         if (err) return done(err);
-
-         if (!user) {
-            // Create an account for the user
-            Role.getRole('USER_ROLE', function (err, role) {
-               if (err) return done(err);
-
-               let email = null;
-               if (profile[strategy.settings.profile.email]) {
-                  if (Array.isArray(profile[strategy.settings.profile.email])) {
-                     email = profile[strategy.settings.profile.email].find(email => {
-                        email.verified === true
-                     });
-                  } else {
-                     email = profile[strategy.settings.profile.email];
-                  }
-               } else {
-                  log.warn(`OAuth2 user profile does not contain email property named ${strategy.settings.profile.email}`);
-                  log.debug(JSON.stringify(profile));
-               }
-
-               const user = {
-                  username: profileId,
-                  displayName: profile[strategy.settings.profile.displayName] || profileId,
-                  email: email,
-                  active: false,
-                  roleId: role._id,
-                  authentication: {
-                     type: strategy.type,
-                     id: profileId,
-                     authenticationConfiguration: {
-                        name: strategy.name
-                     }
-                  }
-               };
-               // TODO: users-next
-               new api.User().create(user).then(newUser => {
-                  if (!newUser.authentication.authenticationConfiguration.enabled) {
-                     log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
-                     return done(null, false, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-                  }
-                  return done(null, newUser);
-               }).catch(err => done(err));
-            });
-         } else if (!user.active) {
-            return done(null, user, { message: "User is not approved, please contact your MAGE administrator to approve your account." });
-         } else if (!user.authentication.authenticationConfiguration.enabled) {
-            log.warn(user.authentication.authenticationConfiguration.title + " authentication is not enabled");
-            return done(null, user, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-         } else {
-            return done(null, user);
-         }
-      });
-   }));
+    });
+  }));
 }
 
 function setDefaults(strategy) {
-   if (!strategy.settings.profile) {
-      strategy.settings.profile = {};
-   }
-   if (!strategy.settings.profile.displayName) {
-      strategy.settings.profile.displayName = 'displayName';
-   }
-   if (!strategy.settings.profile.email) {
-      strategy.settings.profile.email = 'email';
-   }
-   if (!strategy.settings.profile.id) {
-      strategy.settings.profile.id = 'id';
-   }
+  if (!strategy.settings.profile) {
+    strategy.settings.profile = {};
+  }
+  if (!strategy.settings.profile.displayName) {
+    strategy.settings.profile.displayName = 'displayName';
+  }
+  if (!strategy.settings.profile.email) {
+    strategy.settings.profile.email = 'email';
+  }
+  if (!strategy.settings.profile.id) {
+    strategy.settings.profile.id = 'id';
+  }
 }
 
 function initialize(strategy) {
-   setDefaults(strategy);
-
-   // TODO lets test with newer geoaxis server to see if this is still needed
-   // If it is, this should be a admin client side option, would also need to modify the
-   // renderer to provide a more generic message
-   strategy.redirect = false;
-   configure(strategy);
-
-   function authenticate(req, res, next) {
-      passport.authenticate(strategy.name, function (err, user, info = {}) {
-         if (err) return next(err);
-
-         req.user = user;
-
-         // For inactive or disabled accounts don't generate an authorization token
-         if (!user.active || !user.enabled) {
-            log.warn('Failed user login attempt: User ' + user.username + ' account is inactive or disabled.');
-            return next();
-         }
-
-         if (!user.authentication.authenticationConfigurationId) {
-            log.warn('Failed user login attempt: ' + user.authentication.type + ' is not configured');
-            return next();
-         }
-
-         if (!user.authentication.authenticationConfiguration.enabled) {
-            log.warn('Failed user login attempt: Authentication ' + user.authentication.authenticationConfiguration.title + ' is disabled.');
-            return next();
-         }
-
-         tokenService.generateToken(user._id.toString(), TokenAssertion.Authorized, 60 * 5)
-            .then(token => {
-               req.token = token;
-               req.user = user;
-               req.info = info;
-               next();
-            }).catch(err => next(err));
-      })(req, res, next);
-   }
-
-   app.get(`/auth/${strategy.name}/signin`,
-      function (req, res, next) {
-         passport.authenticate(strategy.name, {
-            scope: strategy.settings.scope,
-            state: req.query.state
-         })(req, res, next);
+  setDefaults(strategy);
+
+  // TODO lets test with newer geoaxis server to see if this is still needed
+  // If it is, this should be a admin client side option, would also need to modify the
+  // renderer to provide a more generic message
+  strategy.redirect = false;
+  configure(strategy);
+
+  function authenticate(req, res, next) {
+    passport.authenticate(strategy.name, function (err, user, info = {}) {
+      if (err) return next(err);
+
+      req.user = user;
+
+      // For inactive or disabled accounts don't generate an authorization token
+      if (!user.active || !user.enabled) {
+        log.warn('Failed user login attempt: User ' + user.username + ' account is inactive or disabled.');
+        return next();
       }
-   );
-
-   app.get(`/auth/${strategy.name}/callback`,
-      authenticate,
-      function (req, res) {
-         if (req.query.state === 'mobile') {
-            let uri;
-            if (!req.user.active || !req.user.enabled) {
-               uri = `mage://app/invalid_account?active=${req.user.active}&enabled=${req.user.enabled}`;
-            } else {
-               uri = `mage://app/authentication?token=${req.token}`
-            }
 
-            if (strategy.redirect) {
-               res.redirect(uri);
-            } else {
-               res.render('oauth', { uri: uri });
-            }
-         } else {
-            res.render('authentication', { host: req.getRoot(), success: true, login: { token: req.token, user: req.user } });
-         }
+      if (!user.authentication.authenticationConfigurationId) {
+        log.warn('Failed user login attempt: ' + user.authentication.type + ' is not configured');
+        return next();
+      }
+
+      if (!user.authentication.authenticationConfiguration.enabled) {
+        log.warn('Failed user login attempt: Authentication ' + user.authentication.authenticationConfiguration.title + ' is disabled.');
+        return next();
+      }
+
+      tokenService.generateToken(user._id.toString(), TokenAssertion.Authorized, 60 * 5)
+      .then(token => {
+        req.token = token;
+        req.user = user;
+        req.info = info;
+        next();
+      }).catch(err => next(err));
+    })(req, res, next);
+  }
+
+  app.get(`/auth/${strategy.name}/signin`,
+    function (req, res, next) {
+      passport.authenticate(strategy.name, {
+        scope: strategy.settings.scope,
+        state: req.query.state
+      })(req, res, next);
+    }
+  );
+
+  app.get(`/auth/${strategy.name}/callback`,
+    authenticate,
+    function (req, res) {
+      if (req.query.state === 'mobile') {
+        let uri;
+        if (!req.user.active || !req.user.enabled) {
+          uri = `mage://app/invalid_account?active=${req.user.active}&enabled=${req.user.enabled}`;
+        } else {
+          uri = `mage://app/authentication?token=${req.token}`
+        }
+
+        if (strategy.redirect) {
+          res.redirect(uri);
+        } else {
+          res.render('oauth', { uri: uri });
+        }
+      } else {
+        res.render('authentication', { host: req.getRoot(), success: true, login: { token: req.token, user: req.user } });
       }
-   );
+    }
+  );
 };
 
 module.exports = {
-   initialize
+  initialize
 }
\ No newline at end of file

From cfc9a5a3343e7891b01317f26a4ace46f5557c13 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 9 Sep 2024 13:30:52 -0600
Subject: [PATCH 084/183] refactor(service): users/auth: wip: oauth2 typescript
 transition: add typedefs

---
 service/npm-shrinkwrap.json | 28 ++++++++++++++++++++++++++++
 service/package.json        |  2 ++
 2 files changed, 30 insertions(+)

diff --git a/service/npm-shrinkwrap.json b/service/npm-shrinkwrap.json
index effece5e4..3aad25a50 100644
--- a/service/npm-shrinkwrap.json
+++ b/service/npm-shrinkwrap.json
@@ -73,6 +73,7 @@
         "@fluffy-spoon/substitute": "^1.196.0",
         "@types/archiver": "^5.3.4",
         "@types/async": "^3.0.5",
+        "@types/base-64": "^1.0.2",
         "@types/bson": "^1.0.11",
         "@types/busboy": "^1.5.0",
         "@types/chai": "^4.2.19",
@@ -91,6 +92,7 @@
         "@types/passport": "^1.0.3",
         "@types/passport-http-bearer": "^1.0.41",
         "@types/passport-local": "^1.0.38",
+        "@types/passport-oauth2": "^1.4.17",
         "@types/sinon": "^9.0.4",
         "@types/sinon-chai": "^3.2.4",
         "@types/superagent": "^8.1.3",
@@ -2463,6 +2465,12 @@
       "integrity": "sha512-8iHVLHsCCOBKjCF2KwFe0p9Z3rfM9mL+sSP8btyR5vTjJRAqpBYD28/ZLgXPf0pjG1VxOvtCV/BgXkQbpSe8Hw==",
       "dev": true
     },
+    "node_modules/@types/base-64": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@types/base-64/-/base-64-1.0.2.tgz",
+      "integrity": "sha512-uPgKMmM9fmn7I+Zi6YBqctOye4SlJsHKcisjHIMWpb2YKZRc36GpKyNuQ03JcT+oNXg1m7Uv4wU94EVltn8/cw==",
+      "dev": true
+    },
     "node_modules/@types/body-parser": {
       "version": "1.19.5",
       "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz",
@@ -2720,6 +2728,15 @@
         "form-data": "^4.0.0"
       }
     },
+    "node_modules/@types/oauth": {
+      "version": "0.9.5",
+      "resolved": "https://registry.npmjs.org/@types/oauth/-/oauth-0.9.5.tgz",
+      "integrity": "sha512-+oQ3C2Zx6ambINOcdIARF5Z3Tu3x//HipE889/fqo3sgpQZbe9c6ExdQFtN6qlhpR7p83lTZfPJt0tCAW29dog==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/passport": {
       "version": "1.0.16",
       "resolved": "https://registry.npmjs.org/@types/passport/-/passport-1.0.16.tgz",
@@ -2750,6 +2767,17 @@
         "@types/passport-strategy": "*"
       }
     },
+    "node_modules/@types/passport-oauth2": {
+      "version": "1.4.17",
+      "resolved": "https://registry.npmjs.org/@types/passport-oauth2/-/passport-oauth2-1.4.17.tgz",
+      "integrity": "sha512-ODiAHvso6JcWJ6ZkHHroVp05EHGhqQN533PtFNBkg8Fy5mERDqsr030AX81M0D69ZcaMvhF92SRckEk2B0HYYg==",
+      "dev": true,
+      "dependencies": {
+        "@types/express": "*",
+        "@types/oauth": "*",
+        "@types/passport": "*"
+      }
+    },
     "node_modules/@types/passport-strategy": {
       "version": "0.2.38",
       "resolved": "https://registry.npmjs.org/@types/passport-strategy/-/passport-strategy-0.2.38.tgz",
diff --git a/service/package.json b/service/package.json
index b6123637b..0d030eaab 100644
--- a/service/package.json
+++ b/service/package.json
@@ -90,6 +90,7 @@
     "@fluffy-spoon/substitute": "^1.196.0",
     "@types/archiver": "^5.3.4",
     "@types/async": "^3.0.5",
+    "@types/base-64": "^1.0.2",
     "@types/bson": "^1.0.11",
     "@types/busboy": "^1.5.0",
     "@types/chai": "^4.2.19",
@@ -108,6 +109,7 @@
     "@types/passport": "^1.0.3",
     "@types/passport-http-bearer": "^1.0.41",
     "@types/passport-local": "^1.0.38",
+    "@types/passport-oauth2": "^1.4.17",
     "@types/sinon": "^9.0.4",
     "@types/sinon-chai": "^3.2.4",
     "@types/superagent": "^8.1.3",

From 9a3b0ab1aa140a9bb6288cac67474f74e2fe1397 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 9 Sep 2024 14:32:57 -0600
Subject: [PATCH 085/183] style(service): users/auth: wip: oauth2 typescript:
 white space and types

---
 service/src/ingress/ingress.protocol.oauth.ts | 173 +++++++++---------
 1 file changed, 86 insertions(+), 87 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.oauth.ts b/service/src/ingress/ingress.protocol.oauth.ts
index 79d980839..2df3b3d8e 100644
--- a/service/src/ingress/ingress.protocol.oauth.ts
+++ b/service/src/ingress/ingress.protocol.oauth.ts
@@ -18,7 +18,7 @@ class OAuth2ProfileStrategy extends OAuth2Strategy {
   private _profileURL: string
 
   constructor(options: MageOAuth2Options, verify: VerifyFunction) {
-    super(options, verify)
+    super(options as OAuth2Options, verify)
     if (!options.profileURL) {
       throw new TypeError('OAuth2: missing profileURL')
     }
@@ -26,7 +26,7 @@ class OAuth2ProfileStrategy extends OAuth2Strategy {
     this._oauth2.useAuthorizationHeaderforGET(true)
   }
 
-  async userProfile(accessToken: string, done: (err: unknown, profile?: any) => void): Promise<void> {
+  userProfile(accessToken: string, done: (err: unknown, profile?: any) => void): void {
     this._oauth2.get(this._profileURL, accessToken, (err, body) => {
       if (err) {
         return done(new InternalOAuthError('error fetching oauth2 user profile', err))
@@ -49,95 +49,94 @@ class OAuth2ProfileStrategy extends OAuth2Strategy {
 }
 
 function configure(strategy) {
-  log.info('Configuring ' + strategy.title + ' authentication');
-
-  let customHeaders = null;
-
-  if (strategy.settings.headers) {
-    customHeaders = {};
-    if (strategy.settings.headers.basic) {
-      customHeaders['Authorization'] = `Basic ${base64.encode(`${strategy.settings.clientID}:${strategy.settings.clientSecret}`)}`;
-    }
-  }
-
-  passport.use(strategy.name, new OAuth2ProfileStrategy({
-    clientID: strategy.settings.clientID,
-    clientSecret: strategy.settings.clientSecret,
-    callbackURL: `/auth/${strategy.name}/callback`,
-    authorizationURL: strategy.settings.authorizationURL,
-    tokenURL: strategy.settings.tokenURL,
-    profileURL: strategy.settings.profileURL,
-    customHeaders: customHeaders,
-    scope: strategy.settings.scope,
-    pkce: strategy.settings.pkce,
-    store: true
-  }, function (accessToken, refreshToken, profileResponse, done) {
-    const profile = profileResponse.json;
-
-    if (!profile[strategy.settings.profile.id]) {
-      log.warn("JSON: " + JSON.stringify(profile) + " RAW: " + profileResponse.raw);
-      return done(`OAuth2 user profile does not contain id property named ${strategy.settings.profile.id}`);
-    }
-
-    const profileId = profile[strategy.settings.profile.id];
-
-    // TODO: users-next
-    // TODO: should be by strategy name, not strategy type
-    User.getUserByAuthenticationStrategy(strategy.type, profileId, function (err, user) {
-      if (err) return done(err);
-
-      if (!user) {
-        // Create an account for the user
-        Role.getRole('USER_ROLE', function (err, role) {
-          if (err) return done(err);
+  log.info(`configuring ${strategy.title} oauth2 authentication`);
+  const customHeaders = strategy.settings.headers?.basic ? {
+    authorization: `Basic ${base64.encode(`${strategy.settings.clientID}:${strategy.settings.clientSecret}`)}`
+  } : undefined
+  passport.use(strategy.name, new OAuth2ProfileStrategy(
+    {
+      clientID: strategy.settings.clientID,
+      clientSecret: strategy.settings.clientSecret,
+      callbackURL: `/auth/${strategy.name}/callback`,
+      authorizationURL: strategy.settings.authorizationURL,
+      tokenURL: strategy.settings.tokenURL,
+      profileURL: strategy.settings.profileURL,
+      customHeaders: customHeaders,
+      scope: strategy.settings.scope,
+      pkce: strategy.settings.pkce,
+      /**
+       * cast to `any` because `@types/passport-oauth2` incorrectly does not allow `boolean` for the `store` entry
+       * https://github.com/jaredhanson/passport-oauth2/blob/master/lib/strategy.js#L107
+       */
+      store: true as any
+    },
+    function (accessToken, refreshToken, profileResponse, done) {
+      const profile = profileResponse.json;
+
+      if (!profile[strategy.settings.profile.id]) {
+        log.warn("JSON: " + JSON.stringify(profile) + " RAW: " + profileResponse.raw);
+        return done(`OAuth2 user profile does not contain id property named ${strategy.settings.profile.id}`);
+      }
 
-          let email = null;
-          if (profile[strategy.settings.profile.email]) {
-            if (Array.isArray(profile[strategy.settings.profile.email])) {
-              email = profile[strategy.settings.profile.email].find(email => {
-                email.verified === true
-              });
+      const profileId = profile[strategy.settings.profile.id];
+
+      // TODO: users-next
+      // TODO: should be by strategy name, not strategy type
+      User.getUserByAuthenticationStrategy(strategy.type, profileId, function (err, user) {
+        if (err) return done(err);
+
+        if (!user) {
+          // Create an account for the user
+          Role.getRole('USER_ROLE', function (err, role) {
+            if (err) return done(err);
+
+            let email = null;
+            if (profile[strategy.settings.profile.email]) {
+              if (Array.isArray(profile[strategy.settings.profile.email])) {
+                email = profile[strategy.settings.profile.email].find(email => {
+                  email.verified === true
+                });
+              } else {
+                email = profile[strategy.settings.profile.email];
+              }
             } else {
-              email = profile[strategy.settings.profile.email];
+              log.warn(`OAuth2 user profile does not contain email property named ${strategy.settings.profile.email}`);
+              log.debug(JSON.stringify(profile));
             }
-          } else {
-            log.warn(`OAuth2 user profile does not contain email property named ${strategy.settings.profile.email}`);
-            log.debug(JSON.stringify(profile));
-          }
-
-          const user = {
-            username: profileId,
-            displayName: profile[strategy.settings.profile.displayName] || profileId,
-            email: email,
-            active: false,
-            roleId: role._id,
-            authentication: {
-              type: strategy.type,
-              id: profileId,
-              authenticationConfiguration: {
-                name: strategy.name
+
+            const user = {
+              username: profileId,
+              displayName: profile[strategy.settings.profile.displayName] || profileId,
+              email: email,
+              active: false,
+              roleId: role._id,
+              authentication: {
+                type: strategy.type,
+                id: profileId,
+                authenticationConfiguration: {
+                  name: strategy.name
+                }
               }
-            }
-          };
-          // TODO: users-next
-          new api.User().create(user).then(newUser => {
-            if (!newUser.authentication.authenticationConfiguration.enabled) {
-              log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
-              return done(null, false, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-            }
-            return done(null, newUser);
-          }).catch(err => done(err));
-        });
-      } else if (!user.active) {
-        return done(null, user, { message: "User is not approved, please contact your MAGE administrator to approve your account." });
-      } else if (!user.authentication.authenticationConfiguration.enabled) {
-        log.warn(user.authentication.authenticationConfiguration.title + " authentication is not enabled");
-        return done(null, user, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-      } else {
-        return done(null, user);
-      }
-    });
-  }));
+            };
+            // TODO: users-next
+            new api.User().create(user).then(newUser => {
+              if (!newUser.authentication.authenticationConfiguration.enabled) {
+                log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
+                return done(null, false, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
+              }
+              return done(null, newUser);
+            }).catch(err => done(err));
+          });
+        } else if (!user.active) {
+          return done(null, user, { message: "User is not approved, please contact your MAGE administrator to approve your account." });
+        } else if (!user.authentication.authenticationConfiguration.enabled) {
+          log.warn(user.authentication.authenticationConfiguration.title + " authentication is not enabled");
+          return done(null, user, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
+        } else {
+          return done(null, user);
+        }
+      });
+    }));
 }
 
 function setDefaults(strategy) {

From 333bcb547e09f65ca1d6e62104baeaea323bac68 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 9 Sep 2024 19:12:55 -0600
Subject: [PATCH 086/183] style(service): users/auth: wip: oauth2 typescript:
 more cleanup

---
 service/src/ingress/ingress.protocol.oauth.ts | 201 ++++++++++--------
 1 file changed, 107 insertions(+), 94 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.oauth.ts b/service/src/ingress/ingress.protocol.oauth.ts
index 2df3b3d8e..ccadcd63d 100644
--- a/service/src/ingress/ingress.protocol.oauth.ts
+++ b/service/src/ingress/ingress.protocol.oauth.ts
@@ -3,31 +3,42 @@
 import { InternalOAuthError, Strategy as OAuth2Strategy, StrategyOptions as OAuth2Options, VerifyFunction } from 'passport-oauth2'
 import { TokenAssertion, JWTService } from './verification'
 import base64 from 'base-64'
+import { IdentityProvider } from './entities.authentication'
+import { Authenticator } from 'passport'
 const api = require('../api')
 const log = require('../logger')
 const User = require('../models/user')
 const Role = require('../models/role')
-const { app, passport, tokenService } = require('./index');
 
-interface MageOAuth2Options extends OAuth2Options {
-  profileURL: string
+export type OAuth2ProtocolSettings =
+  Pick<OAuth2Options,
+  | 'clientID'
+  | 'clientSecret'
+  | 'authorizationURL'
+  | 'tokenURL'
+  | 'scope'
+  | 'pkce'
+  > &
+  {
+    profileURL: string,
+    headers?: { basic?: boolean | null | undefined },
+    profile: OAuth2ProfileKeys
+  }
+export type OAuth2ProfileKeys = {
+  id: string
+  email: string
+  displayName: string
 }
 
 class OAuth2ProfileStrategy extends OAuth2Strategy {
 
-  private _profileURL: string
-
-  constructor(options: MageOAuth2Options, verify: VerifyFunction) {
+  constructor(options: OAuth2Options, readonly profileURL: string, verify: VerifyFunction) {
     super(options as OAuth2Options, verify)
-    if (!options.profileURL) {
-      throw new TypeError('OAuth2: missing profileURL')
-    }
-    this._profileURL = options.profileURL
     this._oauth2.useAuthorizationHeaderforGET(true)
   }
 
   userProfile(accessToken: string, done: (err: unknown, profile?: any) => void): void {
-    this._oauth2.get(this._profileURL, accessToken, (err, body) => {
+    this._oauth2.get(this.profileURL, accessToken, (err, body) => {
       if (err) {
         return done(new InternalOAuthError('error fetching oauth2 user profile', err))
       }
@@ -48,95 +59,97 @@ class OAuth2ProfileStrategy extends OAuth2Strategy {
   }
 }
 
-function configure(strategy) {
+function configure(strategy: IdentityProvider, passport: Authenticator, ) {
   log.info(`configuring ${strategy.title} oauth2 authentication`);
-  const customHeaders = strategy.settings.headers?.basic ? {
-    authorization: `Basic ${base64.encode(`${strategy.settings.clientID}:${strategy.settings.clientSecret}`)}`
+  const settings = strategy.protocolSettings as OAuth2ProtocolSettings
+  const customHeaders = settings.headers?.basic ? {
+    authorization: `Basic ${base64.encode(`${settings.clientID}:${settings.clientSecret}`)}`
   } : undefined
-  passport.use(strategy.name, new OAuth2ProfileStrategy(
-    {
-      clientID: strategy.settings.clientID,
-      clientSecret: strategy.settings.clientSecret,
-      callbackURL: `/auth/${strategy.name}/callback`,
-      authorizationURL: strategy.settings.authorizationURL,
-      tokenURL: strategy.settings.tokenURL,
-      profileURL: strategy.settings.profileURL,
-      customHeaders: customHeaders,
-      scope: strategy.settings.scope,
-      pkce: strategy.settings.pkce,
-      /**
-       * cast to `any` because `@types/passport-oauth2` incorrectly does not allow `boolean` for the `store` entry
-       * https://github.com/jaredhanson/passport-oauth2/blob/master/lib/strategy.js#L107
-       */
-      store: true as any
-    },
-    function (accessToken, refreshToken, profileResponse, done) {
-      const profile = profileResponse.json;
-
-      if (!profile[strategy.settings.profile.id]) {
-        log.warn("JSON: " + JSON.stringify(profile) + " RAW: " + profileResponse.raw);
-        return done(`OAuth2 user profile does not contain id property named ${strategy.settings.profile.id}`);
-      }
+  const strategyOptions: OAuth2Options = {
+    clientID: settings.clientID,
+    clientSecret: settings.clientSecret,
+    callbackURL: `/auth/${strategy.name}/callback`,
+    authorizationURL: settings.authorizationURL,
+    tokenURL: settings.tokenURL,
+    customHeaders: customHeaders,
+    scope: settings.scope,
+    pkce: settings.pkce,
+    /**
+     * cast to `any` because `@types/passport-oauth2` incorrectly does not allow `boolean` for the `store` entry
+     * https://github.com/jaredhanson/passport-oauth2/blob/master/lib/strategy.js#L107
+     */
+    store: true as any
+  }
+  const verify: VerifyFunction = (accessToken, refreshToken, profileResponse, done) => {
+    const profile = profileResponse.json
+    const profileKeys = settings.profile
+    if (!profile[profileKeys.id]) {
+      log.warn("JSON: " + JSON.stringify(profile) + " RAW: " + profileResponse.raw);
+      return done(`OAuth2 user profile does not contain id property named ${profileKeys.id}`);
+    }
 
-      const profileId = profile[strategy.settings.profile.id];
-
-      // TODO: users-next
-      // TODO: should be by strategy name, not strategy type
-      User.getUserByAuthenticationStrategy(strategy.type, profileId, function (err, user) {
-        if (err) return done(err);
-
-        if (!user) {
-          // Create an account for the user
-          Role.getRole('USER_ROLE', function (err, role) {
-            if (err) return done(err);
-
-            let email = null;
-            if (profile[strategy.settings.profile.email]) {
-              if (Array.isArray(profile[strategy.settings.profile.email])) {
-                email = profile[strategy.settings.profile.email].find(email => {
-                  email.verified === true
-                });
-              } else {
-                email = profile[strategy.settings.profile.email];
-              }
+    const profileId = profile[settings.profile.id];
+
+    // TODO: users-next
+    // TODO: should be by strategy name, not strategy type
+    User.getUserByAuthenticationStrategy(strategy.type, profileId, function (err, user) {
+      if (err) return done(err);
+
+      if (!user) {
+        // Create an account for the user
+        Role.getRole('USER_ROLE', function (err, role) {
+          if (err) {
+            return done(err)
+          }
+          const profileEmail = profile[profileKeys.email]
+          if (profile[profileKeys.email]) {
+            if (Array.isArray(profile[profileKeys.email])) {
+              email = profile[profileKeys.email].find(email => {
+                email.verified === true
+              });
             } else {
-              log.warn(`OAuth2 user profile does not contain email property named ${strategy.settings.profile.email}`);
-              log.debug(JSON.stringify(profile));
+              email = profile[settings.profile.email];
             }
-
-            const user = {
-              username: profileId,
-              displayName: profile[strategy.settings.profile.displayName] || profileId,
-              email: email,
-              active: false,
-              roleId: role._id,
-              authentication: {
-                type: strategy.type,
-                id: profileId,
-                authenticationConfiguration: {
-                  name: strategy.name
-                }
-              }
-            };
-            // TODO: users-next
-            new api.User().create(user).then(newUser => {
-              if (!newUser.authentication.authenticationConfiguration.enabled) {
-                log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
-                return done(null, false, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
+          } else {
+            log.warn(`OAuth2 user profile does not contain email property named ${profileKeys.email}`);
+            log.debug(JSON.stringify(profile));
+          }
+
+          const user = {
+            username: profileId,
+            displayName: profile[profileKeys.displayName] || profileId,
+            email: email,
+            active: false,
+            roleId: role._id,
+            authentication: {
+              type: strategy.type,
+              id: profileId,
+              authenticationConfiguration: {
+                name: strategy.name
               }
-              return done(null, newUser);
-            }).catch(err => done(err));
-          });
-        } else if (!user.active) {
-          return done(null, user, { message: "User is not approved, please contact your MAGE administrator to approve your account." });
-        } else if (!user.authentication.authenticationConfiguration.enabled) {
-          log.warn(user.authentication.authenticationConfiguration.title + " authentication is not enabled");
-          return done(null, user, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-        } else {
-          return done(null, user);
-        }
-      });
-    }));
+            }
+          };
+          // TODO: users-next
+          new api.User().create(user).then(newUser => {
+            if (!newUser.authentication.authenticationConfiguration.enabled) {
+              log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
+              return done(null, false, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
+            }
+            return done(null, newUser);
+          }).catch(err => done(err));
+        });
+      } else if (!user.active) {
+        return done(null, user, { message: "User is not approved, please contact your MAGE administrator to approve your account." });
+      } else if (!user.authentication.authenticationConfiguration.enabled) {
+        log.warn(user.authentication.authenticationConfiguration.title + " authentication is not enabled");
+        return done(null, user, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
+      } else {
+        return done(null, user);
+      }
+    });
+  }
+  const oauth2Strategy = new OAuth2ProfileStrategy(strategyOptions, verify)
+
 }
 
 function setDefaults(strategy) {

From 3a818fab6aaecd7c981b1fe5ea85001707ef407b Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 11 Sep 2024 11:58:54 -0600
Subject: [PATCH 087/183] style(service): users/auth: wip: remove unused
 imports and other cleanup

---
 service/src/models/team.js     | 1 -
 service/src/models/user.js     | 8 --------
 service/src/provision/index.js | 3 +--
 service/src/routes/index.js    | 1 +
 4 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/service/src/models/team.js b/service/src/models/team.js
index c15cbde94..d1ba70e7f 100644
--- a/service/src/models/team.js
+++ b/service/src/models/team.js
@@ -1,4 +1,3 @@
-/// <reference path="./user.d.ts" />
 var mongoose = require('mongoose')
   , async = require('async')
   , Event = require('./event')
diff --git a/service/src/models/user.js b/service/src/models/user.js
index 5d3645a77..e8f63c518 100644
--- a/service/src/models/user.js
+++ b/service/src/models/user.js
@@ -1,15 +1,7 @@
 "use strict";
 
 const mongoose = require('mongoose')
-  , async = require('async')
   , moment = require('moment')
-  , Token = require('./token')
-  , Login = require('./login')
-  , Event = require('./event')
-  , Team = require('./team')
-  , Observation = require('./observation')
-  , Location = require('./location')
-  , CappedLocation = require('./cappedLocation')
   , Authentication = require('./authentication')
   , AuthenticationConfiguration = require('./authenticationconfiguration')
   , { pageQuery } = require('../adapters/base/adapters.base.db.mongoose')
diff --git a/service/src/provision/index.js b/service/src/provision/index.js
index 060cdef16..c83981e30 100644
--- a/service/src/provision/index.js
+++ b/service/src/provision/index.js
@@ -1,4 +1,3 @@
-const Setting = require('../models/setting');
 const { modulesPathsInDir } = require('../utilities/loader');
 const log = require('../logger');
 const AuthenticationConfiguration = require('../models/authenticationconfiguration');
@@ -46,7 +45,7 @@ Provision.prototype.check = function (type, name, options) {
         next();
       });
     }).catch(err => {
-      next(err);;
+      next(err);
     });
   };
 };
diff --git a/service/src/routes/index.js b/service/src/routes/index.js
index 2787b0b8b..439b5a80f 100644
--- a/service/src/routes/index.js
+++ b/service/src/routes/index.js
@@ -101,6 +101,7 @@ module.exports = function(app, security) {
   // Grab the feature for any endpoint that uses observationId
   app.param('observationId', function(req, res, next, observationId) {
     req.observationId = observationId;
+    // TODO: obs types: use repo
     new api.Observation(req.event).getById(observationId, function(
       err,
       observation

From e9e3b09d19aeef92baf6f47efe893c89e68d9819 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 11 Sep 2024 16:30:52 -0600
Subject: [PATCH 088/183] refactor(service): users/auth: wip: rename
 authentication entities module to ingress; added unique index on session
 token

---
 .../ingress/adapters.authentication.db.mongoose.ts  | 13 +++++++++----
 ...tities.authentication.ts => ingress.entities.ts} | 11 +++++++++--
 2 files changed, 18 insertions(+), 6 deletions(-)
 rename service/src/ingress/{entities.authentication.ts => ingress.entities.ts} (91%)

diff --git a/service/src/ingress/adapters.authentication.db.mongoose.ts b/service/src/ingress/adapters.authentication.db.mongoose.ts
index 0e33445c7..56eecedec 100644
--- a/service/src/ingress/adapters.authentication.db.mongoose.ts
+++ b/service/src/ingress/adapters.authentication.db.mongoose.ts
@@ -2,7 +2,7 @@ import crypto from 'crypto'
 import mongoose, { Schema } from 'mongoose'
 import { UserDocumentExpanded } from '../adapters/users/adapters.users.db.mongoose'
 import { UserId } from '../entities/users/entities.users'
-import { Session, SessionRepository } from './entities.authentication'
+import { Session, SessionRepository } from './ingress.entities'
 
 export interface SessionDocument {
   token: string
@@ -26,7 +26,7 @@ const SessionSchema = new Schema<SessionDocument, SessionModel>(
 )
 
 // TODO: index token
-SessionSchema.index({ token: 1 })
+SessionSchema.index({ token: 1, unique: 1 })
 SessionSchema.index({ expirationDate: 1 }, { expireAfterSeconds: 0 })
 
 const populateSessionUserRole: mongoose.PopulateOptions = {
@@ -65,8 +65,13 @@ export function createSessionRepository(conn: mongoose.Connection, collectionNam
       return await model.findOneAndUpdate(query, update,
         { upsert: true, new: true, populate: populateSessionUserRole })
     },
-    async removeSession(token: string): Promise<void> {
-      await model.deleteOne({ token })
+    async removeSession(token: string): Promise<Session | null> {
+      const session = await this.findSessionByToken(token)
+      if (!session) {
+        return null
+      }
+      const removed = await this.model.deleteOne({ token })
+      return removed.deletedCount === 1 ? session : null
     },
     async removeSessionsForUser(userId: UserId): Promise<number> {
       const { deletedCount } = await model.deleteMany({ userId: new mongoose.Types.ObjectId(userId) })
diff --git a/service/src/ingress/entities.authentication.ts b/service/src/ingress/ingress.entities.ts
similarity index 91%
rename from service/src/ingress/entities.authentication.ts
rename to service/src/ingress/ingress.entities.ts
index 0ee5bbc29..d5e8103bf 100644
--- a/service/src/ingress/entities.authentication.ts
+++ b/service/src/ingress/ingress.entities.ts
@@ -33,6 +33,8 @@ export interface AuthenticationProtocol {
   name: string
 }
 
+export type IdentityProviderId = string
+
 /**
  * An identity provider (IDP) is a service maintains user profiles and that Mage trusts to authenticate user
  * credentials via a specific authentication protocol.  Mage delegates user authentication to identity providers.
@@ -40,6 +42,7 @@ export interface AuthenticationProtocol {
  * user profile.
  */
 export interface IdentityProvider {
+  id: IdentityProviderId
   name: string
   title: string
   protocol: AuthenticationProtocol
@@ -47,6 +50,7 @@ export interface IdentityProvider {
   enabled: boolean
   lastUpdated: Date
   enrollmentPolicy: EnrollmentPolicy
+  deviceEnrollmentPolicy: DeviceEnrollmentPolicy
   description?: string | null
   textColor?: string | null
   buttonColor?: string | null
@@ -62,11 +66,14 @@ export interface EnrollmentPolicy {
   assignToTeams: TeamId[]
   assignToEvents: MageEventId[]
   requireAccountApproval: boolean
-  // TODO: move to different policy?
+
+}
+
+export interface DeviceEnrollmentPolicy {
   requireDeviceApproval: boolean
 }
 
 export interface IdentityProviderRepository {
-  findById(): Promise<IdentityProvider | null>
+  findById(id: IdentityProviderId): Promise<IdentityProvider | null>
   findByName(name: string): Promise<IdentityProvider | null>
 }

From 0018ee53af5773a9cc0ad72cc9a3eb5757a06013 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 11 Sep 2024 16:32:39 -0600
Subject: [PATCH 089/183] style(service): users/auth: fix renamed references in
 devices web controller

---
 .../adapters/devices/adapters.devices.controllers.web.ts    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/service/src/adapters/devices/adapters.devices.controllers.web.ts b/service/src/adapters/devices/adapters.devices.controllers.web.ts
index 1e13635aa..69353bb1a 100644
--- a/service/src/adapters/devices/adapters.devices.controllers.web.ts
+++ b/service/src/adapters/devices/adapters.devices.controllers.web.ts
@@ -1,10 +1,10 @@
 import express from 'express'
 import { entityNotFound, invalidInput } from '../../app.api/app.api.errors'
 import { DevicePermissionService } from '../../app.api/devices/app.api.devices'
-import { SessionRepository } from '../../authentication/entities.authentication'
 import { Device, DeviceRepository, FindDevicesSpec } from '../../entities/devices/entities.devices'
 import { PageOf, PagingParameters } from '../../entities/entities.global'
 import { User, UserFindParameters, UserRepository } from '../../entities/users/entities.users'
+import { SessionRepository } from '../../ingress/ingress.entities'
 import { compatibilityMageAppErrorHandler, WebAppRequestFactory } from '../adapters.controllers.web'
 
 
@@ -58,7 +58,7 @@ export function DeviceRoutes(deviceRepo: DeviceRepository, userRepo: UserReposit
         try {
           if (update.registered === false) {
             console.info(`update device ${idInPath} to unregistered`)
-            const sessionsRemovedCount = await sessionRepo.removeSessionForDevice(idInPath)
+            const sessionsRemovedCount = await sessionRepo.removeSessionsForDevice(idInPath)
             console.info(`removed ${sessionsRemovedCount} session(s) for device ${idInPath}`)
           }
           const updated = await deviceRepo.update({ ...update, id: idInPath })
@@ -79,7 +79,7 @@ export function DeviceRoutes(deviceRepo: DeviceRepository, userRepo: UserReposit
           const idInPath = req.params.id
           console.info(`delete device`, idInPath)
           const deleted = await deviceRepo.removeById(idInPath)
-          const removedSessionsCount = sessionRepo.removeSessionForDevice(idInPath)
+          const removedSessionsCount = sessionRepo.removeSessionsForDevice(idInPath)
           console.info(`removed ${removedSessionsCount} session(s) for device ${idInPath}`)
           // TODO: the old observation model had a middleware that removed the device id from created observations,
           // but do we really care that much

From 10501f7bbea89d3a058f3d4093404ba74051be6a Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 11 Sep 2024 16:33:30 -0600
Subject: [PATCH 090/183] style(service): users/auth: comment on authentication
 model

---
 service/src/models/authentication.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/service/src/models/authentication.js b/service/src/models/authentication.js
index c30288121..018041540 100644
--- a/service/src/models/authentication.js
+++ b/service/src/models/authentication.js
@@ -12,6 +12,7 @@ const Schema = mongoose.Schema;
 
 const AuthenticationSchema = new Schema(
   {
+    // TODO: type is really not necessary
     type: { type: String, required: true },
     id: { type: String, required: false },
     authenticationConfigurationId: { type: Schema.Types.ObjectId, ref: 'AuthenticationConfiguration', required: false }

From 54f1292a7d90db0c4352868f0b056aceba38f7d1 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 11 Sep 2024 16:37:30 -0600
Subject: [PATCH 091/183] refactor(service): users/auth: rename authentication
 db adapter to sessions

---
 ...entication.db.mongoose.ts => sessions.adapters.db.mongoose.ts} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/src/ingress/{adapters.authentication.db.mongoose.ts => sessions.adapters.db.mongoose.ts} (100%)

diff --git a/service/src/ingress/adapters.authentication.db.mongoose.ts b/service/src/ingress/sessions.adapters.db.mongoose.ts
similarity index 100%
rename from service/src/ingress/adapters.authentication.db.mongoose.ts
rename to service/src/ingress/sessions.adapters.db.mongoose.ts

From 8c47475cbf22c9341c09746d7d7707993b559fcf Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 11 Sep 2024 22:15:57 -0600
Subject: [PATCH 092/183] refactor(service): users/auth: move authentication
 configuration model to ingress identity provider adapter

---
 .../identity-providers.adapters.db.mongoose.ts}                   | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/src/{models/authenticationconfiguration.js => ingress/identity-providers.adapters.db.mongoose.ts} (100%)

diff --git a/service/src/models/authenticationconfiguration.js b/service/src/ingress/identity-providers.adapters.db.mongoose.ts
similarity index 100%
rename from service/src/models/authenticationconfiguration.js
rename to service/src/ingress/identity-providers.adapters.db.mongoose.ts

From 82be3fdb33491f5e4ffe848012132a83fed95e2c Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 11 Sep 2024 22:53:52 -0600
Subject: [PATCH 093/183] refactor(service): users/auth: move authentication
 model to ingress local idp adapter

---
 .../local-idp.adapters.db.mongoose.ts}                            | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/src/{models/authentication.js => ingress/local-idp.adapters.db.mongoose.ts} (100%)

diff --git a/service/src/models/authentication.js b/service/src/ingress/local-idp.adapters.db.mongoose.ts
similarity index 100%
rename from service/src/models/authentication.js
rename to service/src/ingress/local-idp.adapters.db.mongoose.ts

From e224d8a13305219423a06a0ad0058319b1b316e0 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 11 Sep 2024 23:20:22 -0600
Subject: [PATCH 094/183] refactor(service): users/auth: add local idp mongoose
 repository stub

---
 .../ingress/local-idp.adapters.db.mongoose.ts | 47 +++++++++++++++----
 1 file changed, 39 insertions(+), 8 deletions(-)

diff --git a/service/src/ingress/local-idp.adapters.db.mongoose.ts b/service/src/ingress/local-idp.adapters.db.mongoose.ts
index 018041540..5fcd38f2b 100644
--- a/service/src/ingress/local-idp.adapters.db.mongoose.ts
+++ b/service/src/ingress/local-idp.adapters.db.mongoose.ts
@@ -1,12 +1,14 @@
 "use strict";
 
-const mongoose = require('mongoose')
-  , async = require('async')
-  , hasher = require('../utilities/pbkdf2')()
-  , User = require('./user')
-  , Token = require('./token')
-  , AuthenticationConfiguration = require('./authenticationconfiguration')
-  , PasswordValidator = require('../utilities/passwordValidator');
+import mongoose from 'mongoose'
+import { LocalIdpAccount, LocalIdpRepository } from './local-idp.entities'
+
+const async = require('async')
+const hasher = require('../utilities/pbkdf2')()
+const User = require('./user')
+const Token = require('./token')
+// TODO: users-next
+const PasswordValidator = require('../utilities/passwordValidator');
 
 const Schema = mongoose.Schema;
 
@@ -209,4 +211,33 @@ exports.updateAuthentication = function (authentication) {
 
 exports.removeAuthenticationById = function (authenticationId, done) {
   Authentication.findByIdAndRemove(authenticationId, done);
-};
\ No newline at end of file
+};
+
+export class LocalIdpMongooseRepository implements LocalIdpRepository {
+
+  constructor(private UserModel: UserModel, private AuthenticationModel: AuthenticationModel) {
+
+  }
+
+  createLocalAccount(account: LocalIdpAccount): Promise<LocalIdpAccount> {
+    throw new Error('Method not implemented.')
+  }
+
+  async readLocalAccount(id: LocalIdpAccountId): Promise<LocalIdpAccount | null> {
+    const dbId = new mongoose.Types.ObjectId(id)
+    const userModelInstance = await this.UserModel.findOne({ username })
+      .populate({ path: 'authenticationId', populate: 'authenticationConfigurationId' })
+    if (!userModelInstance) {
+      return null
+    }
+    return userModelInstance.authenticationId
+  }
+
+  updateLocalAccount(update: Partial<LocalIdpAccount> & Pick<LocalIdpAccount, 'username'>): Promise<LocalIdpAccount | null> {
+    throw new Error('Method not implemented.')
+  }
+
+  deleteLocalAccount(id: string): Promise<LocalIdpAccount | null> {
+    throw new Error('Method not implemented.')
+  }
+}
\ No newline at end of file

From 88eee2a35e74fb330d93e876cffcfb25d7b1f631 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 12 Sep 2024 07:10:47 -0600
Subject: [PATCH 095/183] refactor(service): users/auth: remove obsolete user
 model functions

---
 service/src/models/user.js | 93 +-------------------------------------
 1 file changed, 1 insertion(+), 92 deletions(-)

diff --git a/service/src/models/user.js b/service/src/models/user.js
index e8f63c518..7290f75c5 100644
--- a/service/src/models/user.js
+++ b/service/src/models/user.js
@@ -3,9 +3,8 @@
 const mongoose = require('mongoose')
   , moment = require('moment')
   , Authentication = require('./authentication')
+  // TODO: users-next
   , AuthenticationConfiguration = require('./authenticationconfiguration')
-  , { pageQuery } = require('../adapters/base/adapters.base.db.mongoose')
-  , { pageOf } = require('../entities/entities.global')
   , FilterParser = require('../utilities/filterParser');
 
 
@@ -15,36 +14,6 @@ exports.transform = DbUserToObject;
 const User = mongoose.model('User', UserSchema);
 exports.Model = User;
 
-exports.getUserById = function (id, callback) {
-  let result = User.findById(id).populate('roleId').populate({ path: 'authenticationId', populate: { path: 'authenticationConfigurationId' } });
-  if (typeof callback === 'function') {
-    result = result.then(
-      user => {
-        callback(null, user);
-      },
-      err => {
-        callback(err);
-      });
-  }
-  return result;
-};
-
-exports.getUserByUsername = function (username, callback) {
-  User.findOne({ username: username.toLowerCase() }).populate('roleId').populate({ path: 'authenticationId', populate: { path: 'authenticationConfigurationId' } }).exec(callback);
-};
-
-exports.getUserByAuthenticationId = function (id, callback) {
-  User.findOne({ authenticationId: id }).populate('roleId').populate({ path: 'authenticationId', populate: { path: 'authenticationConfigurationId' } }).exec(callback);
-}
-
-exports.getUserByAuthenticationStrategy = function (strategy, uid, callback) {
-  Authentication.getAuthenticationByStrategy(strategy, uid, function (err, authentication) {
-    if (err || !authentication) return callback(err);
-
-    User.findOne({ authenticationId: authentication._id }).populate('roleId').populate({ path: 'authenticationId', populate: { path: 'authenticationConfigurationId' } }).exec(callback);
-  });
-}
-
 function createQueryConditions(filter) {
   const conditions = FilterParser.parse(filter);
 
@@ -74,66 +43,6 @@ exports.count = function (options, callback) {
   });
 };
 
-exports.getUsers = async function (options, callback) {
-  if (typeof options === 'function') {
-    callback = options;
-    options = {};
-  }
-
-  options = options || {};
-  const filter = options.filter || {};
-
-  const conditions = createQueryConditions(filter);
-
-  let baseQuery = User.find(conditions).populate({ path: 'authenticationId', populate: { path: 'authenticationConfigurationId' } });
-
-  if (options.lean) {
-    baseQuery = baseQuery.lean();
-  }
-
-  if (options.populate && (options.populate.indexOf('roleId') !== -1)) {
-    baseQuery = baseQuery.populate('roleId');
-  }
-
-  const isPaging = options.limit != null && options.limit > 0;
-  if (isPaging) {
-    const limit = Math.abs(options.limit) || 10;
-    const start = (Math.abs(options.start) || 0);
-    const page = Math.ceil(start / limit);
-
-    const which = {
-      pageSize: limit,
-      pageIndex: page,
-      includeTotalCount: true
-    };
-    try {
-      const counted = await pageQuery(baseQuery, which);
-      const users = [];
-      for await (const userDoc of counted.query.cursor()) {
-        users.push(entityForDocument(userDoc));
-      }
-      const pageof = pageOf(users, which, counted.totalCount);
-      callback(null, users, pageof);
-    } catch (err) {
-      callback(err);
-    }
-  } else {
-    baseQuery.exec(function (err, users) {
-      callback(err, users, null);
-    });
-  }
-};
-
-function entityForDocument(doc) {
-  const json = doc.toJSON();
-  const entity = {
-    ...json,
-    id: doc._id.toHexString()
-  }
-
-  return entity;
-}
-
 exports.createUser = function (user, callback) {
   Authentication.createAuthentication(user.authentication).then(authentication => {
     const newUser = {

From 21b9d7f418fd8144c9cf76c96502127d524baf1c Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 13 Sep 2024 12:26:24 -0600
Subject: [PATCH 096/183] refactor(service): users/auth: remove references to
 user icon type property which did not serve any real purpose

---
 service/src/api/user.js                       |  6 ++--
 service/src/entities/users/entities.users.ts  | 17 ++--------
 service/src/migrations/007-user-icon.js       | 32 +++++++++++--------
 service/src/routes/users.js                   | 19 ++---------
 .../ng1/admin/users/user.edit.component.js    |  3 +-
 5 files changed, 26 insertions(+), 51 deletions(-)

diff --git a/service/src/api/user.js b/service/src/api/user.js
index 26028f6a3..e07b8b268 100644
--- a/service/src/api/user.js
+++ b/service/src/api/user.js
@@ -4,6 +4,7 @@ const UserModel = require('../models/user')
   , LoginModel = require('../models/login')
   , DeviceModel = require('../models/device')
   , TeamModel = require('../models/team')
+  // TODO: users-next
   , AuthenticationConfiguration = require('../models/authenticationconfiguration')
   , path = require('path')
   , fs = require('fs-extra')
@@ -145,18 +146,15 @@ User.prototype.create = async function (user, options = {}) {
     } catch { }
   }
 
-  if (options.icon && (options.icon.type === 'create' || options.icon.type === 'upload')) {
+  if (options.icon) {
     try {
       const icon = iconPath(newUser._id, newUser, options.icon);
       await fs.move(options.icon.path, icon.absolutePath);
-
-      newUser.icon.type = options.icon.type;
       newUser.icon.relativePath = icon.relativePath;
       newUser.icon.contentType = options.icon.mimetype;
       newUser.icon.size = options.icon.size;
       newUser.icon.text = options.icon.text;
       newUser.icon.color = options.icon.color;
-
       await newUser.save();
     } catch { }
   }
diff --git a/service/src/entities/users/entities.users.ts b/service/src/entities/users/entities.users.ts
index b8080852e..c0651b32e 100644
--- a/service/src/entities/users/entities.users.ts
+++ b/service/src/entities/users/entities.users.ts
@@ -44,15 +44,10 @@ export interface Phone {
 }
 
 /**
- * TODO: There is not much value to retaining the `type`, `text`, and `color` attributes.  Only the web app's user
- * admin screen uses these to set default form values, but the web app always generates a raster png from those values
- * anyway.
+ * A user's icon is the image that appears on the map to indicate the user's location.  If the icon has text and color,
+ * a client can choose to render that rather than the raster image the server stores.
  */
 export interface UserIcon {
-  /**
-   * Type defaults to {@link UserIconType.None} via database layer.
-   */
-  type: UserIconType
   text?: string
   color?: string
   contentType?: string
@@ -60,12 +55,6 @@ export interface UserIcon {
   relativePath?: string
 }
 
-export enum UserIconType {
-  None = 'none',
-  Upload = 'upload',
-  Create = 'create',
-}
-
 export interface Avatar {
   contentType?: string,
   size?: number,
@@ -83,7 +72,7 @@ export interface UserRepository {
   findAllByIds(ids: UserId[]): Promise<{ [id: string]: User | null }>
   find<MappedResult>(which?: UserFindParameters, mapping?: (user: User) => MappedResult): Promise<PageOf<MappedResult>>
   saveMapIcon(userId: UserId, icon: UserIcon, content: NodeJS.ReadableStream | Buffer): Promise<User | UserRepositoryError>
-  saveAvatar(avatar: Avatar, content: NodeJS.ReadableStream | Buffer): Promise<User | UserRepositoryError>
+  saveAvatar(userId: UserId, avatar: Avatar, content: NodeJS.ReadableStream | Buffer): Promise<User | UserRepositoryError>
   deleteMapIcon(userId: UserId): Promise<User | UserRepositoryError>
   deleteAvatar(userId: UserId): Promise<User | UserRepositoryError>
 }
diff --git a/service/src/migrations/007-user-icon.js b/service/src/migrations/007-user-icon.js
index dea8b81a3..0c01114ac 100644
--- a/service/src/migrations/007-user-icon.js
+++ b/service/src/migrations/007-user-icon.js
@@ -1,23 +1,27 @@
-const async = require('async')
-  , User = require('../models/user');
+// const async = require('async')
+//   , User = require('../models/user');
 
 exports.id = '007-user-icon';
 
+/**
+ * This migration became obsolete after removing the `type` property from the user icon document.
+ */
 exports.up = function(done) {
-  this.log('updating user icons');
+  done();
 
-  // TODO: users-next
-  User.getUsers(function(err, users) {
-    if (err) return done(err);
+  // this.log('updating user icons');
+  // // TODO: users-next
+  // User.getUsers(function(err, users) {
+  //   if (err) return done(err);
 
-    async.each(users, function(user, done) {
-      user.icon = user.icon || {};
-      user.icon.type = user.icon.relativePath ? 'upload' : 'none';
-      user.save(done);
-    }, function(err) {
-      done(err);
-    });
-  });
+  //   async.each(users, function(user, done) {
+  //     user.icon = user.icon || {};
+  //     user.icon.type = user.icon.relativePath ? 'upload' : 'none';
+  //     user.save(done);
+  //   }, function(err) {
+  //     done(err);
+  //   });
+  // });
 };
 
 exports.down = function(done) {
diff --git a/service/src/routes/users.js b/service/src/routes/users.js
index e283ee19d..9383f1132 100644
--- a/service/src/routes/users.js
+++ b/service/src/routes/users.js
@@ -45,30 +45,15 @@ module.exports = function (app, security) {
 
   function parseIconUpload(req, res, next) {
     let iconMetadata = req.param('iconMetadata') || {};
-    if (typeof iconMetadata === 'string' || iconMetadata instanceof String) {
+    if (typeof iconMetadata === 'string') {
       iconMetadata = JSON.parse(iconMetadata);
     }
-
     const files = req.files || {};
-    let [icon] = files.icon || [];
+    let [ icon ] = files.icon || [];
     if (icon) {
-      // default type to upload
-      if (!iconMetadata.type) iconMetadata.type = 'upload';
-
-      if (iconMetadata.type !== 'create' && iconMetadata.type !== 'upload') {
-        return res.status(400).send('invalid icon metadata');
-      }
-
-      icon.type = iconMetadata.type;
       icon.text = iconMetadata.text;
       icon.color = iconMetadata.color;
-    } else if (iconMetadata.type === 'none') {
-      icon = {
-        type: 'none'
-      };
-      files.icon = [icon];
     }
-
     next();
   }
 
diff --git a/web-app/src/ng1/admin/users/user.edit.component.js b/web-app/src/ng1/admin/users/user.edit.component.js
index df386c74c..eb6004e3d 100644
--- a/web-app/src/ng1/admin/users/user.edit.component.js
+++ b/web-app/src/ng1/admin/users/user.edit.component.js
@@ -56,9 +56,8 @@ class AdminUserEditController {
     if (this.$stateParams.userId) {
       this.UserService.getUser(this.$stateParams.userId).then(user => {
         this.user = angular.copy(user);
-
         this.iconMetadata = {
-          type: this.user.icon.type,
+          type: this.user.icon.type || 'upload',
           text: this.user.icon.text,
           color: this.user.icon.color
         };

From 17a80ce1192daeeea0ebe5983467725f05c58393 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 13 Sep 2024 13:13:29 -0600
Subject: [PATCH 097/183] refactor(service): users/auth: add todo tag comments
 for removing legacy authentication configuration model references

---
 service/src/provision/index.js                           | 1 +
 service/src/routes/authenticationconfigurations.js       | 1 +
 service/src/routes/setup.js                              | 1 +
 .../src/security/utilities/secure-property-appender.js   | 3 ++-
 service/src/transformers/authenticationconfiguration.js  | 1 +
 service/src/utilities/authenticationApiAppender.js       | 9 +++++----
 6 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/service/src/provision/index.js b/service/src/provision/index.js
index c83981e30..96e3057cb 100644
--- a/service/src/provision/index.js
+++ b/service/src/provision/index.js
@@ -1,5 +1,6 @@
 const { modulesPathsInDir } = require('../utilities/loader');
 const log = require('../logger');
+// TODO: users-next
 const AuthenticationConfiguration = require('../models/authenticationconfiguration');
 
 /**
diff --git a/service/src/routes/authenticationconfigurations.js b/service/src/routes/authenticationconfigurations.js
index cdf239035..1439cf27a 100644
--- a/service/src/routes/authenticationconfigurations.js
+++ b/service/src/routes/authenticationconfigurations.js
@@ -4,6 +4,7 @@ const log = require('winston')
   , AsyncLock = require('async-lock')
   , access = require('../access')
   , Authentication = require('../models/authentication')
+  // TODO: users-next
   , AuthenticationConfiguration = require('../models/authenticationconfiguration')
   , AuthenticationConfigurationTransformer = require('../transformers/authenticationconfiguration')
   , SecretStoreService = require('../security/secret-store-service')
diff --git a/service/src/routes/setup.js b/service/src/routes/setup.js
index ae2462f72..21c1182e0 100644
--- a/service/src/routes/setup.js
+++ b/service/src/routes/setup.js
@@ -4,6 +4,7 @@ module.exports = function (app, security) {
     , User = require('../models/user')
     , Device = require('../models/device')
     , userTransformer = require('../transformers/user')
+    // TODO: users-next
     , AuthenticationConfiguration = require('../models/authenticationconfiguration');
 
   function authorizeSetup(req, res, next) {
diff --git a/service/src/security/utilities/secure-property-appender.js b/service/src/security/utilities/secure-property-appender.js
index 3a05c1115..b4208c370 100644
--- a/service/src/security/utilities/secure-property-appender.js
+++ b/service/src/security/utilities/secure-property-appender.js
@@ -1,11 +1,12 @@
 'use strict';
 
 const SecretStoreService = require('../secret-store-service')
+// TODO: users-next
     , AuthenticationConfiguration = require('../../models/authenticationconfiguration')
 
 /**
  * Helper function to append secure properties to a configuration under the settings property.
- * 
+ *
  * @param {*} config Must contain the _id property
  * @returns A copy of the config with secure properties appended (if any exist)
  */
diff --git a/service/src/transformers/authenticationconfiguration.js b/service/src/transformers/authenticationconfiguration.js
index ed596dfd6..cc068c954 100644
--- a/service/src/transformers/authenticationconfiguration.js
+++ b/service/src/transformers/authenticationconfiguration.js
@@ -1,5 +1,6 @@
 "use strict";
 
+// TODO: users-next
 const AuthenticationConfiguration = require('../models/authenticationconfiguration');
 
 function transformAuthenticationConfiguration(authenticationConfiguration, options) {
diff --git a/service/src/utilities/authenticationApiAppender.js b/service/src/utilities/authenticationApiAppender.js
index 715af906f..b437ce7e7 100644
--- a/service/src/utilities/authenticationApiAppender.js
+++ b/service/src/utilities/authenticationApiAppender.js
@@ -1,15 +1,16 @@
 "use strict";
 
 const extend = require('util')._extend
+    // TODO: users-next
     , AuthenticationConfiguration = require('../models/authenticationconfiguration')
     , AuthenticationConfigurationTransformer = require('../transformers/authenticationconfiguration');
 
 /**
- * Appends authenticationStrategies to the config.api object (the original config.api is not modified; this method returns a copy).  
+ * Appends authenticationStrategies to the config.api object (the original config.api is not modified; this method returns a copy).
  * These strategies are read from the db.
- * 
- * @param {*} api 
- * @param {*} options 
+ *
+ * @param {*} api
+ * @param {*} options
  * @returns Promise containing the modified copy of the api parameter
  */
 async function append(api, options) {

From 2f28e378540441c14568bfb58c7e8c79b9cbfcbe Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 13 Sep 2024 13:28:31 -0600
Subject: [PATCH 098/183] refactor(service): users/auth: remove references to
 user icon type property from openapi doc

---
 service/src/docs/openapi.yaml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/service/src/docs/openapi.yaml b/service/src/docs/openapi.yaml
index f2912c75e..80d6271df 100644
--- a/service/src/docs/openapi.yaml
+++ b/service/src/docs/openapi.yaml
@@ -3081,7 +3081,6 @@ components:
         iconMetadata:
           type: object
           properties:
-            type: { $ref: '#/components/schemas/UserIcon/properties/type' }
             color: { $ref: '#/components/schemas/UserIcon/properties/color' }
             text: { $ref: '#/components/schemas/UserIcon/properties/text' }
         icon:
@@ -3115,13 +3114,6 @@ components:
         location.
       type: object
       properties:
-        type:
-          description: The origin of the icon
-          type: string
-          enum:
-            - create
-            - upload
-            - none
         color:
           description: Color is only applicable for `create` type icons.
           $ref: '#/components/schemas/ColorHex'

From e94be622c8587e1c849670611d08e96d7e426d59 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 16 Sep 2024 15:10:23 -0600
Subject: [PATCH 099/183] refactor(service): users/auth: move password
 utilities to typescript

---
 service/src/utilities/{pbkdf2.js => password-hashing.ts}          | 0
 .../src/utilities/{passwordValidator.js => password-policy.ts}    | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename service/src/utilities/{pbkdf2.js => password-hashing.ts} (100%)
 rename service/src/utilities/{passwordValidator.js => password-policy.ts} (100%)

diff --git a/service/src/utilities/pbkdf2.js b/service/src/utilities/password-hashing.ts
similarity index 100%
rename from service/src/utilities/pbkdf2.js
rename to service/src/utilities/password-hashing.ts
diff --git a/service/src/utilities/passwordValidator.js b/service/src/utilities/password-policy.ts
similarity index 100%
rename from service/src/utilities/passwordValidator.js
rename to service/src/utilities/password-policy.ts

From ea2527a7da4ebefb9bfe0242c2688919de0cc2e1 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 17 Sep 2024 07:11:11 -0600
Subject: [PATCH 100/183] refactor(service): users/auth: password hash utility
 typescript conversion BREAKING CHANGE: upgraded password hash digest to
 sha512 - will need a db migration and users will need to reset passwords

---
 service/src/utilities/password-hashing.ts | 128 ++++++++++++----------
 1 file changed, 69 insertions(+), 59 deletions(-)

diff --git a/service/src/utilities/password-hashing.ts b/service/src/utilities/password-hashing.ts
index 4f172bcbc..0dee32bfe 100644
--- a/service/src/utilities/password-hashing.ts
+++ b/service/src/utilities/password-hashing.ts
@@ -1,88 +1,98 @@
-module.exports = function(options) {
-
-  const crypto = require('crypto');
-
-  options = options || {};
-  const iterations = options.iterations || 12000;
-  const saltLength = options.saltLength || 128;
-  const derivedKeyLength = options.derivedKeyLength || 256;
+import crypto from 'crypto'
+import util from 'util'
+
+const digest = 'sha512'
+const pbkdf2_promise = util.promisify(crypto.pbkdf2)
+
+export type HashedPassword = {
+  salt: string
+  derivedKey: string
+  derivedKeyLength: number
+  iterations: number
+}
+
+export function formatHashedPassword(hashed: HashedPassword): string {
+  return `${hashed.salt}::${hashed.derivedKey}::${hashed.derivedKeyLength}::${hashed.iterations}`
+}
+
+export type PasswordHashOptions = {
+  iterations?: number
+  saltLength?: number
+  derivedKeyLength?: number
+}
+
+export const defaultPasswordHashOptions = {
+  iterations: 12000,
+  saltLength: 128,
+  derivedKeyLength: 256,
+} as const
+
+export class PasswordHashUtil {
+
+  private iterations: number
+  private saltLength: number
+  private derivedKeyLength: number
+
+  constructor(readonly options = defaultPasswordHashOptions) {
+    this.iterations = options.iterations || defaultPasswordHashOptions.iterations
+    this.saltLength = options.saltLength || defaultPasswordHashOptions.saltLength
+    this.derivedKeyLength = options.derivedKeyLength || defaultPasswordHashOptions.derivedKeyLength
+  }
 
   /**
    * Serialize a password object containing all the information needed to check a password into a string
    * The info is salt, derivedKey, derivedKey length and number of iterations
    */
-  function serializePassword(password) {
-    return password.salt + "::" +
-           password.derivedKey + "::" +
-           password.derivedKeyLength + "::" +
-           password.iterations;
+  serializePassword(hashed: HashedPassword): string {
+    return formatHashedPassword(hashed)
   }
 
   /**
-   * Deserialize a string into a password object
+   * Deserialize a string into a password object.
    * The info is salt, derivedKey, derivedKey length and number of iterations
    */
-  function deserializePassword(password) {
-    const items = password.split('::');
-
+  deserializePassword(password: string): HashedPassword {
+    const items = password.split('::')
     return {
       salt: items[0],
       derivedKey: items[1],
       derivedKeyLength: parseInt(items[2], 10),
       iterations: parseInt(items[3], 10)
-    };
+    }
   }
-
   /**
-   * Hash a password using node.js' crypto's PBKDF2
+   * Hash a password using Node crypto's PBKDF2
    * Description here: http://en.wikipedia.org/wiki/PBKDF2
    * Number of iterations are saved in case we change the setting in the future
-   * @param {String} password
-   * @param {Function} callback Signature: err, hashedPassword
    */
-  function hashPassword(password, callback) {
-    const salt = crypto.randomBytes(saltLength).toString('base64');
-
-    crypto.pbkdf2(password, salt, iterations, derivedKeyLength, 'sha1', function (err, derivedKey) {
-      if (err) { return callback(err); }
-
-      const hashedPassword = serializePassword({
-        salt: salt,
-        iterations: iterations,
-        derivedKeyLength: derivedKeyLength,
-        derivedKey: derivedKey.toString('base64')
-      });
-
-      callback(null, hashedPassword);
-    });
+  async hashPassword(password: string): Promise<HashedPassword> {
+    const salt = crypto.randomBytes(this.saltLength).toString('base64')
+    // TODO: upgrade hash algorithm
+    const derivedKey = await pbkdf2_promise(password, salt, this.iterations, this.derivedKeyLength, digest)
+    return {
+      salt,
+      iterations: this.iterations,
+      derivedKeyLength: this.derivedKeyLength,
+      derivedKey: derivedKey.toString('base64')
+    }
   }
 
   /**
-   * Compare a password to a hashed password
-   * @param {String} password
-   * @param {String} hashedPassword
-   * @param {Function} callback Signature: err, true/false
+   * Compare a password to a password hash
    */
-  function validPassword(password, hashedPassword, callback) {
-    if (!hashedPassword) return callback(false);
-
-    hashedPassword = deserializePassword(hashedPassword);
-
-    if (!hashedPassword.salt || !hashedPassword.derivedKey || !hashedPassword.iterations || !hashedPassword.derivedKeyLength) {
-      return callback(new Error("hashedPassword doesn't have the right format"));
+  async validPassword(password: string, serializedHash: string): Promise<boolean> {
+    if (!serializedHash) {
+      return false
+    }
+    const hash = this.deserializePassword(serializedHash)
+    if (!hash.salt || !hash.derivedKey || !hash.iterations || !hash.derivedKeyLength) {
+      throw new Error('invalid password hash')
     }
+    const testHash = await pbkdf2_promise(password, hash.salt, hash.iterations, hash.derivedKeyLength, digest)
+    return testHash.toString('base64') === hash.derivedKey
+  }
+}
 
-    // Use the hashedPassword password's parameters to hash the candidate password
-    crypto.pbkdf2(password, hashedPassword.salt, hashedPassword.iterations, hashedPassword.derivedKeyLength, 'sha1', function (err, derivedKey) {
-      if (err) { return callback(err); }
 
-      callback(null, derivedKey.toString('base64') === hashedPassword.derivedKey);
-    });
-  }
 
-  return {
-    hashPassword: hashPassword,
-    validPassword: validPassword
-  };
 
-};

From aa58e7e928ef7444f4ef0f467ad3ab27d3bb267a Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 17 Sep 2024 07:27:35 -0600
Subject: [PATCH 101/183] style(service): white space

---
 service/src/utilities/password-hashing.ts | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/service/src/utilities/password-hashing.ts b/service/src/utilities/password-hashing.ts
index 0dee32bfe..6db47e70d 100644
--- a/service/src/utilities/password-hashing.ts
+++ b/service/src/utilities/password-hashing.ts
@@ -92,7 +92,3 @@ export class PasswordHashUtil {
     return testHash.toString('base64') === hash.derivedKey
   }
 }
-
-
-
-

From 6b5f4cd35817f18c4c4e4c378e1b9cf66e016f95 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 17 Sep 2024 07:31:37 -0600
Subject: [PATCH 102/183] refactor(service): users/auth: export default
 password hash utility

---
 service/src/utilities/password-hashing.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/service/src/utilities/password-hashing.ts b/service/src/utilities/password-hashing.ts
index 6db47e70d..215a57069 100644
--- a/service/src/utilities/password-hashing.ts
+++ b/service/src/utilities/password-hashing.ts
@@ -92,3 +92,5 @@ export class PasswordHashUtil {
     return testHash.toString('base64') === hash.derivedKey
   }
 }
+
+export const defaultHashUtil = new PasswordHashUtil()
\ No newline at end of file

From 463d22182ba463054a74c559429a4fc4a38527c5 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 17 Sep 2024 14:52:59 -0600
Subject: [PATCH 103/183] refactor(service): users/auth: password policy
 utility typescript transition

---
 service/src/utilities/password-policy.ts | 272 +++++++++++------------
 1 file changed, 124 insertions(+), 148 deletions(-)

diff --git a/service/src/utilities/password-policy.ts b/service/src/utilities/password-policy.ts
index 7b0b82087..d6e4ea55b 100644
--- a/service/src/utilities/password-policy.ts
+++ b/service/src/utilities/password-policy.ts
@@ -1,190 +1,166 @@
-const util = require('util')
-  , log = require('winston')
-  , hasher = require('./pbkdf2')();
+import { defaultHashUtil } from './password-hashing'
+
+const SPECIAL_CHARS = '~!@#$%^&*(),.?":{}|<>_=;-'
+
+export type PasswordRequirements = {
+  passwordMinLengthEnabled: boolean
+  passwordMinLength: number
+  /** Minimum number of alpha characters, A-Z, case-insensitive */
+  minCharsEnabled: boolean
+  minChars: number
+  maxConCharsEnabled: boolean
+  maxConChars: number
+  lowLettersEnabled: boolean
+  lowLetters: number
+  highLettersEnabled: boolean
+  highLetters: number
+  numbersEnabled: boolean
+  numbers: number
+  specialCharsEnabled: boolean
+  specialChars: number
+  restrictSpecialCharsEnabled: boolean
+  restrictSpecialChars: string
+  passwordHistoryCountEnabled: boolean
+  passwordHistoryCount: number
+  helpText: string
+}
 
-const SPECIAL_CHARS = '~!@#$%^&*(),.?":{}|<>_=;-';
-const validPassword = util.promisify(hasher.validPassword);
+export type PasswordValidationResult = {
+  valid: boolean
+  errorMsg: string | null
+}
 
-async function validate(passwordPolicy, { password, previousPasswords: previousHashedPasswords }) {
+export async function validate(policy: PasswordRequirements, { password, previousPasswords }: { password: string, previousPasswords: string[] }): Promise<PasswordValidationResult> {
   if (!password) {
     return {
       valid: false,
       errorMsg: 'Password is missing'
-    };
+    }
   }
-
-  const invalid = 
-    !validatePasswordLength(passwordPolicy, password) ||
-    !validateMinimumCharacters(passwordPolicy, password) ||
-    !validateMaximumConsecutiveCharacters(passwordPolicy, password) ||
-    !validateMinimumLowercaseCharacters(passwordPolicy, password) ||
-    !validateMinimumUppercaseCharacters(passwordPolicy, password) ||
-    !validateMinimumNumbers(passwordPolicy, password) ||
-    !validateMinimumSpecialCharacters(passwordPolicy, password) ||
-    !(await validatePasswordHistory(passwordPolicy, password, previousHashedPasswords));
-
-  return {
-    valid: !invalid,
-    errorMsg: invalid ? passwordPolicy.helpText : null
-  };
+  const valid =
+    validatePasswordLength(policy, password) &&
+    validateMinimumCharacters(policy, password) &&
+    validateMaximumConsecutiveCharacters(policy, password) &&
+    validateMinimumLowercaseCharacters(policy, password) &&
+    validateMinimumUppercaseCharacters(policy, password) &&
+    validateMinimumNumbers(policy, password) &&
+    validateMinimumSpecialCharacters(policy, password) &&
+    (await validatePasswordHistory(policy, password, previousPasswords))
+  return { valid, errorMsg: valid ? null : policy.helpText }
 }
 
-function validatePasswordLength(passwordPolicy, password) {
-  let isValid = true;
-  if (passwordPolicy.passwordMinLengthEnabled) {
-    isValid = password.length >= passwordPolicy.passwordMinLength;
-  }
-
-  log.debug('Password meets min length: ' + isValid);
-
-  return isValid;
+function validatePasswordLength(policy: PasswordRequirements, password: string): boolean {
+  return policy.passwordMinLengthEnabled &&
+    password.length >= policy.passwordMinLength
 }
 
-function validateMinimumCharacters(passwordPolicy, password) {
-  let isValid = true;
-  if (passwordPolicy.minCharsEnabled) {
-    let passwordCount = 0;
-    for (let i = 0; i < password.length; i++) {
-      const a = password[i];
-
-      if (a.match(/[a-z]/i)) {
-        passwordCount++;
-      }
+function validateMinimumCharacters(policy: PasswordRequirements, password: string): boolean {
+  if (!policy.minCharsEnabled) {
+    return true
+  }
+  let letterCount = 0
+  for (let i = 0; i < password.length; i++) {
+    if (password[i].match(/[a-z]/i)) {
+      letterCount++
     }
-
-    isValid = passwordCount >= passwordPolicy.minChars;
-    log.debug('Password meets miniminum letters: ' + isValid);
   }
-  return isValid;
+  return letterCount >= policy.minChars
 }
 
-function validateMaximumConsecutiveCharacters(passwordPolicy, password) {
-  let isValid = true;
-  if (passwordPolicy.maxConCharsEnabled) {
-    let conCount = 0;
-    for (let i = 0; i < password.length; i++) {
-      const a = password[i];
-
-      if (a.match(/[a-z]/i)) {
-        conCount++;
-      } else {
-        conCount = 0;
-      }
-
-      if (conCount > passwordPolicy.maxConChars) {
-        isValid = false;
-        break;
-      }
-    }
-    log.debug('Password meets max consecutive letters: ' + isValid);
+function validateMaximumConsecutiveCharacters(policy: PasswordRequirements, password: string): boolean {
+  if (!policy.maxConCharsEnabled) {
+    return true
   }
-  return isValid;
+  const tooManyConsecutiveLetters = new RegExp(`[a-z]{${policy.maxConChars + 1}}`, 'i')
+  return !tooManyConsecutiveLetters.test(password)
 }
 
-function validateMinimumLowercaseCharacters(passwordPolicy, password) {
-  let isValid = true;
-  if (passwordPolicy.lowLettersEnabled) {
-    let passwordCount = 0;
-    for (let i = 0; i < password.length; i++) {
-      const a = password[i];
-
-      if (a.match(/[a-z]/)) {
-        passwordCount++;
-      }
+function validateMinimumLowercaseCharacters(policy: PasswordRequirements, password: string): boolean {
+  if (!policy.lowLettersEnabled) {
+    return true
+  }
+  let letterCount = 0
+  for (let i = 0; i < password.length; i++) {
+    if (/[a-z]/.test(password[i])) {
+      letterCount++
     }
-    isValid = passwordCount >= passwordPolicy.lowLetters;
-    log.debug('Password meets minimum lowercase letters: ' + isValid);
   }
-  return isValid;
+  return letterCount >= policy.lowLetters
 }
 
-function validateMinimumUppercaseCharacters(passwordPolicy, password) {
-  let isValid = true;
-  if (passwordPolicy.highLettersEnabled) {
-    let passwordCount = 0;
-    for (let i = 0; i < password.length; i++) {
-      const a = password[i];
-
-      if (a.match(/[A-Z]/)) {
-        passwordCount++;
-      }
+function validateMinimumUppercaseCharacters(policy: PasswordRequirements, password: string): boolean {
+  if (!policy.highLettersEnabled) {
+    return true
+  }
+  let letterCount = 0
+  for (let i = 0; i < password.length; i++) {
+    if (/[A-Z]/.test(password[i])) {
+      letterCount++
     }
-    isValid = passwordCount >= passwordPolicy.highLetters;
-    log.debug('Password meets minimum uppercase letters: ' + isValid);
   }
-  return isValid;
+  return letterCount >= policy.highLetters
 }
 
-function validateMinimumNumbers(passwordPolicy, password) {
-  let isValid = true;
-  if (passwordPolicy.numbersEnabled) {
-    let passwordCount = 0;
-    for (let i = 0; i < password.length; i++) {
-      let a = password[i];
-
-      if (a.match(/[0-9]/)) {
-        passwordCount++;
-      }
+function validateMinimumNumbers(policy: PasswordRequirements, password: string): boolean {
+  if (!policy.numbersEnabled) {
+    return true
+  }
+  let numberCount = 0
+  for (let i = 0; i < password.length; i++) {
+    if (/[0-9]/.test(password[i])) {
+      numberCount++
     }
-    isValid = passwordCount >= passwordPolicy.numbers;
-    log.debug('Password meets minimum numbers: ' + isValid);
   }
-  return isValid;
+  return numberCount >= policy.numbers
 }
 
-function validateMinimumSpecialCharacters(passwordPolicy, password) {
-  let isValid = true;
-  if (passwordPolicy.specialCharsEnabled) {
-    let regex = null;
-    let nonAllowedRegex = null;
-    if (passwordPolicy.restrictSpecialCharsEnabled) {
-      nonAllowedRegex = new RegExp('[' + createRestrictedRegex(passwordPolicy.restrictSpecialChars) + ']');
-      regex = new RegExp('[' + passwordPolicy.restrictSpecialChars + ']');
-    } else {
-      regex = new RegExp('[' + SPECIAL_CHARS + ']');
+function validateMinimumSpecialCharacters(policy: PasswordRequirements, password: string): boolean {
+  if (!policy.specialCharsEnabled) {
+    return true
+  }
+  let allowedChars = null
+  let forbiddenChars = null
+  if (policy.restrictSpecialCharsEnabled) {
+    forbiddenChars = new RegExp('[' + createRestrictedRegex(policy.restrictSpecialChars) + ']')
+    allowedChars = new RegExp('[' + policy.restrictSpecialChars + ']')
+  }
+  else {
+    allowedChars = new RegExp('[' + SPECIAL_CHARS + ']')
+  }
+  let specialCharCount = 0
+  for (let i = 0; i < password.length && specialCharCount < policy.specialChars && specialCharCount > -1; i++) {
+    const char = password[i]
+    if (forbiddenChars && forbiddenChars.test(char)) {
+      specialCharCount = -1
     }
-
-    let specialCharCount = 0;
-    for (let i = 0; i < password.length; i++) {
-      const a = password[i];
-
-      if (nonAllowedRegex && a.match(nonAllowedRegex)) {
-        specialCharCount = -1;
-        break;
-      }
-
-      if (a.match(regex)) {
-        specialCharCount++;
-      }
+    else if (allowedChars.test(char)) {
+      specialCharCount++
     }
-    isValid = specialCharCount >= passwordPolicy.specialChars;
-    log.debug('Password meets special characters policy: ' + isValid);
   }
-  return isValid;
+  return specialCharCount >= policy.specialChars
 }
 
-function createRestrictedRegex(restrictedChars) {
-  let nonAllowedRegex = '';
-
+function createRestrictedRegex(restrictedChars: string): string {
+  let forbiddenRegex = ''
   for (let i = 0; i < SPECIAL_CHARS.length; i++) {
-    const specialChar = SPECIAL_CHARS[i];
-
+    const specialChar = SPECIAL_CHARS[i]
     if (!restrictedChars.includes(specialChar)) {
-      nonAllowedRegex += specialChar;
+      forbiddenRegex += specialChar
     }
   }
-
-  return nonAllowedRegex;
+  return forbiddenRegex
 }
 
-async function validatePasswordHistory(passwordPolicy, password, passwords) {
-  if (!passwordPolicy.passwordHistoryCountEnabled || !passwords) return Promise.resolve(true);
-
-  const policyPasswords = passwords.slice(0, passwordPolicy.passwordHistoryCount);
-  const results = await Promise.all(policyPasswords.map(policyPassword => validPassword(password, policyPassword)));
-  return !results.includes(true);
+async function validatePasswordHistory(policy: PasswordRequirements, password: string, previousPasswords: string[]): Promise<boolean> {
+  if (!policy.passwordHistoryCountEnabled || !previousPasswords) {
+    return true
+  }
+  const truncatedHistory = previousPasswords.slice(0, policy.passwordHistoryCount)
+  for (const previousPasswordHash of truncatedHistory) {
+    const used = await defaultHashUtil.validPassword(password, previousPasswordHash)
+    if (used) {
+      return false
+    }
+  }
+  return true
 }
-
-module.exports = {
-  validate
-}
\ No newline at end of file

From bf1f2b099b6bf20f2a9ecde13cf2432e622c52f4 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 17 Sep 2024 17:55:29 -0600
Subject: [PATCH 104/183] refactor(service): users/auth: add local idp entities

---
 service/src/ingress/local-idp.entities.ts | 53 +++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 service/src/ingress/local-idp.entities.ts

diff --git a/service/src/ingress/local-idp.entities.ts b/service/src/ingress/local-idp.entities.ts
new file mode 100644
index 000000000..4c134057b
--- /dev/null
+++ b/service/src/ingress/local-idp.entities.ts
@@ -0,0 +1,53 @@
+import { PasswordRequirements } from '../utilities/password-policy'
+
+export type LocalIdpAccountId = string
+
+export interface LocalIdpAccount {
+  id: LocalIdpAccountId
+  hashedPassword: string
+  previousHashedPasswords: string[]
+  security: {
+    locked: boolean
+    lockedUntil: Date
+    invalidLoginAttempts: number
+    numberOfTimesLocked: number
+  }
+}
+
+export interface LocalIdpEnrollment {
+  username: string
+  password: string
+  displayName: string
+  email?: string
+  phone?: string
+}
+
+export interface AccountLockPolicy {
+  enabled: boolean
+  /**
+   * The number of failed login attempts allowed before locking the account
+   */
+  lockAfterInvalidLoginCount: number
+  /**
+   * The duration in seconds to lock an account after reaching the failed login threshold
+   */
+  lockDurationSeconds: number
+  /**
+   * The number of account locks allowed before disabling the account
+   */
+  disableAfterLockCount: number
+}
+
+export interface SecurityPolicy {
+  passwordRequirements: PasswordRequirements
+  accountLock: AccountLockPolicy
+}
+
+export interface LocalIdpRepository {
+  readSecurityPolicy(): Promise<SecurityPolicy>
+  updateSecurityPolicy(policy: SecurityPolicy): Promise<SecurityPolicy>
+  createLocalAccount(account: LocalIdpAccount): Promise<LocalIdpAccount>
+  readLocalAccount(id: LocalIdpAccountId): Promise<LocalIdpAccount | null>
+  updateLocalAccount(update: Partial<LocalIdpAccount> & Pick<LocalIdpAccount, 'id'>): Promise<LocalIdpAccount | null>
+  deleteLocalAccount(id: LocalIdpAccountId): Promise<LocalIdpAccount | null>
+}
\ No newline at end of file

From a9cc5a8f734aad88806b1bef046f57ead4e454d6 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 17 Sep 2024 18:00:33 -0600
Subject: [PATCH 105/183] refactor(service): users/auth: move old pbkdf2 test
 to typescript file

---
 service/test/{pbkdf2.js => utilities/password-hashing.test.ts} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/test/{pbkdf2.js => utilities/password-hashing.test.ts} (100%)

diff --git a/service/test/pbkdf2.js b/service/test/utilities/password-hashing.test.ts
similarity index 100%
rename from service/test/pbkdf2.js
rename to service/test/utilities/password-hashing.test.ts

From ae99dd56d938ee248cad86a391f79ad8666f28ba Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 17 Sep 2024 18:08:57 -0600
Subject: [PATCH 106/183] refactor(service): users/auth: convert password hash
 js to typescript

---
 .../test/utilities/password-hashing.test.ts   | 46 +++++++++----------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/service/test/utilities/password-hashing.test.ts b/service/test/utilities/password-hashing.test.ts
index 23e3afcdb..2f480885f 100644
--- a/service/test/utilities/password-hashing.test.ts
+++ b/service/test/utilities/password-hashing.test.ts
@@ -1,32 +1,32 @@
-var crypto = require('crypto')
-  , pbkdf2 = require('../lib/utilities/pbkdf2.js');
+import crypto from 'crypto'
+import { expect } from 'chai'
+import * as PasswordHashing from '../lib/utilities/password-hashing'
 
-describe("PBKDF2 tests", function() {
-  var hasher = new pbkdf2();
+describe('password hashing', function() {
 
-  it("should hash password", function(done) {
-    hasher.hashPassword("password", function(err, hash) {
-      hash.should.be.a('string');
-      var items = hash.split('::');
-      items.should.have.length(4);
-      items[2].should.equal('256');
-      items[3].should.equal('12000');
+  const hasher = PasswordHashing.defaultHashUtil
 
-      done(err);
-    });
-  });
+  it('should hash password', async function() {
+    const hashed = await hasher.hashPassword('password')
+    hashed.should.be.a('string')
+    const items = hashed.split('::')
+    items.should.have.length(4)
+    items[2].should.equal('256')
+    items[3].should.equal('12000')
+  })
 
-  it("should validate password", function(done) {
-    var hash = [
+  it('should validate password', async function() {
+    const hash = [
       crypto.randomBytes(128).toString('base64').slice(0, 128),
       crypto.randomBytes(256).toString('base64'),
       256,
       12000,
-    ].join("::");
+    ].join('::')
+    const valid = await hasher.validPassword('password', hash)
+    expect(valid).to.be.true
+  })
 
-    hasher.validPassword("password", hash, function(err) {
-      done(err);
-    });
-  });
-
-});
+  it('has meaningful tests', async function() {
+    expect.fail('todo')
+  })
+})

From 65ac78b1340f80805276d0e8ea53c25f61768b2a Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 17 Sep 2024 18:34:54 -0600
Subject: [PATCH 107/183] refactor(service): users/auth: add todo comment on
 old test

---
 service/test/utilities/password-hashing.test.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/service/test/utilities/password-hashing.test.ts b/service/test/utilities/password-hashing.test.ts
index 2f480885f..56736aca8 100644
--- a/service/test/utilities/password-hashing.test.ts
+++ b/service/test/utilities/password-hashing.test.ts
@@ -16,14 +16,14 @@ describe('password hashing', function() {
   })
 
   it('should validate password', async function() {
+    // TODO: what is this testing?
     const hash = [
       crypto.randomBytes(128).toString('base64').slice(0, 128),
       crypto.randomBytes(256).toString('base64'),
       256,
       12000,
     ].join('::')
-    const valid = await hasher.validPassword('password', hash)
-    expect(valid).to.be.true
+    await hasher.validPassword('password', hash)
   })
 
   it('has meaningful tests', async function() {

From e6988d7450658e1eccb9a07d05637bf1b7721d14 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 19 Sep 2024 07:43:44 -0600
Subject: [PATCH 108/183] refactor(service): users/auth: implement identity
 provider repository

---
 ...identity-providers.adapters.db.mongoose.ts | 191 +++++++++++++-----
 1 file changed, 145 insertions(+), 46 deletions(-)

diff --git a/service/src/ingress/identity-providers.adapters.db.mongoose.ts b/service/src/ingress/identity-providers.adapters.db.mongoose.ts
index 7b4a0322d..f5f9a19c7 100644
--- a/service/src/ingress/identity-providers.adapters.db.mongoose.ts
+++ b/service/src/ingress/identity-providers.adapters.db.mongoose.ts
@@ -1,11 +1,39 @@
-"use strict";
+import mongoose from 'mongoose'
+import { BaseMongooseRepository } from '../adapters/base/adapters.base.db.mongoose'
+import { MageEventId } from '../entities/events/entities.events'
+import { TeamId } from '../entities/teams/entities.teams'
+import { DeviceEnrollmentPolicy, IdentityProvider, IdentityProviderRepository, UserEnrollmentPolicy } from './ingress.entities'
 
-const mongoose = require('mongoose');
+type ObjectId = mongoose.Types.ObjectId
 
-// Creates a new Mongoose Schema object
-const Schema = mongoose.Schema;
+const Schema = mongoose.Schema
 
-const AuthenticationConfigurationSchema = new Schema(
+export type CommonIdpSettings = {
+  usersReqAdmin?: { enabled: boolean }
+  devicesReqAdmin?: { enabled: boolean }
+  newUserTeams?: TeamId[]
+  newUserEvents?: MageEventId[]
+}
+
+export type IdentityProviderDocument = {
+  _id: ObjectId
+  name: string
+  /**
+   * IDP type maps to an ingress authentication protocol.
+   */
+  type: string
+  lastUpdated: Date
+  enabled: boolean
+  title?: string
+  settings: CommonIdpSettings & Record<string, any>
+  textColor?: string
+  buttonColor?: string
+  icon?: Buffer
+}
+
+export type IdentityProviderModel = mongoose.Model<IdentityProviderDocument>
+
+export const IdentityProviderSchema = new Schema<IdentityProviderDocument>(
   {
     name: { type: String, required: true },
     type: { type: String, required: true },
@@ -21,14 +49,110 @@ const AuthenticationConfigurationSchema = new Schema(
       updatedAt: 'lastUpdated'
     },
     versionKey: false,
-    toObject: {
-      transform: DbAuthenticationConfigurationToObject
+  }
+)
+
+IdentityProviderSchema.index({ name: 1, type: 1 }, { unique: true })
+
+export function idpEntityForDocument(doc: IdentityProviderDocument): IdentityProvider {
+  const settings = doc.settings || {}
+  const userEnrollmentPolicy: UserEnrollmentPolicy = {
+    accountApprovalRequired: !!settings.usersReqAdmin?.enabled,
+    assignToTeams: doc.settings.newUserTeams || [],
+    assignToEvents: doc.settings.newUserEvents || [],
+  }
+  const deviceEnrollmentPolicy: DeviceEnrollmentPolicy = {
+    deviceApprovalRequired: !!settings.devicesReqAdmin?.enabled
+  }
+  return {
+    id: doc._id.toHexString(),
+    name: doc.name,
+    enabled: doc.enabled,
+    lastUpdated: doc.lastUpdated,
+    // TODO: use protocol instance if appropriate
+    protocol: { name: doc.type },
+    title: doc.title || doc.name,
+    protocolSettings: doc.settings,
+    textColor: doc.textColor,
+    buttonColor: doc.buttonColor,
+    icon: doc.icon,
+    userEnrollmentPolicy,
+    deviceEnrollmentPolicy,
+  }
+}
+
+export function idpDocumentForEntity(entity: Partial<IdentityProvider>): Partial<IdentityProviderDocument> {
+  // TODO: maybe delegate to protocol to copy settings
+  const settings = entity.protocolSettings ? { ...entity.protocolSettings } as CommonIdpSettings : undefined
+  const { userEnrollmentPolicy, deviceEnrollmentPolicy } = entity
+  if (settings && userEnrollmentPolicy) {
+    settings.usersReqAdmin = { enabled: !!userEnrollmentPolicy?.accountApprovalRequired }
+    settings.newUserTeams = userEnrollmentPolicy?.assignToTeams || []
+    settings.newUserEvents = userEnrollmentPolicy?.assignToEvents || []
+  }
+  if (settings && deviceEnrollmentPolicy) {
+    settings.devicesReqAdmin = { enabled: !!deviceEnrollmentPolicy?.deviceApprovalRequired }
+  }
+  const doc = {} as Partial<IdentityProviderDocument>
+  const entityHasKey = (key: keyof IdentityProvider): boolean => Object.prototype.hasOwnProperty.call(entity, key)
+  if (entityHasKey('id')) {
+    doc._id = new mongoose.Types.ObjectId(entity.id)
+  }
+  if (entityHasKey('buttonColor')) {
+    doc.buttonColor = entity.buttonColor
+  }
+  if (entityHasKey('enabled')) {
+    doc.enabled = entity.enabled
+  }
+  if (entityHasKey('icon')) {
+    doc.icon = entity.icon
+  }
+  if (entityHasKey('lastUpdated')) {
+    doc.lastUpdated = entity.lastUpdated ? new Date(entity.lastUpdated) : undefined
+  }
+  if (entityHasKey('name')) {
+    doc.name = entity.name
+  }
+  if (entityHasKey('protocol')) {
+    doc.type = entity.protocol?.name
+  }
+  if (entityHasKey('textColor')) {
+    doc.textColor = entity.textColor
+  }
+  if (entityHasKey('title')) {
+    doc.title = entity.title
+  }
+  return doc
+}
+
+export class IdentityProviderMongooseRepository extends BaseMongooseRepository<IdentityProviderDocument, IdentityProviderModel, IdentityProvider> implements IdentityProviderRepository {
+
+  constructor(model: IdentityProviderModel) {
+    super(model, { docToEntity: idpEntityForDocument, entityToDocStub: idpDocumentForEntity })
+  }
+
+  findIdpById(id: string): Promise<IdentityProvider | null> {
+    return super.findById(id)
+  }
+
+  async findIdpByName(name: string): Promise<IdentityProvider | null> {
+    const doc = await this.model.findOne({ name })
+    if (doc) {
+      return this.entityForDocument(doc)
     }
+    return null
   }
-);
 
-AuthenticationConfigurationSchema.index({ name: 1, type: 1 }, { unique: true });
+  updateIdp(update: Partial<IdentityProvider> & Pick<IdentityProvider, 'id'>): Promise<IdentityProvider | null> {
+    throw new Error('Method not implemented.')
+  }
 
+  deleteIdp(id: string): Promise<IdentityProvider | null> {
+    return super.removeById(id)
+  }
+}
+
+// TODO: should be per protocol and identity provider and in entity layer
 const whitelist = ['name', 'type', 'title', 'textColor', 'buttonColor', 'icon'];
 const blacklist = ['clientsecret', 'bindcredentials', 'privatecert', 'decryptionpvk'];
 const secureMask = '*****';
@@ -59,29 +183,7 @@ function DbAuthenticationConfigurationToObject(config, ret, options) {
   ret.icon = ret.icon ? ret.icon.toString('base64') : null;
 }
 
-exports.transform = DbAuthenticationConfigurationToObject;
-exports.secureMask = secureMask;
-exports.blacklist = blacklist;
-
-const AuthenticationConfiguration = mongoose.model('AuthenticationConfiguration', AuthenticationConfigurationSchema);
-exports.Model = AuthenticationConfiguration;
-
-exports.getById = function (id) {
-  return AuthenticationConfiguration.findById(id).exec();
-};
-
-exports.getConfiguration = function (type, name) {
-  return AuthenticationConfiguration.findOne({ type: type, name: name }).exec();
-};
-
-exports.getConfigurationsByType = function (type) {
-  return AuthenticationConfiguration.find({ type: type }).exec();
-};
-
-exports.getAllConfigurations = function () {
-  return AuthenticationConfiguration.find({}).exec();
-};
-
+// TODO: move to api layer
 function manageIcon(config) {
   if (config.icon) {
     if (config.icon.startsWith('data')) {
@@ -94,6 +196,7 @@ function manageIcon(config) {
   }
 }
 
+// TODO: move to protocol
 function manageSettings(config) {
   if (config.settings.scope) {
     if (!Array.isArray(config.settings.scope)) {
@@ -108,20 +211,16 @@ function manageSettings(config) {
 
 //TODO move the 'manage' methods to a pre save method
 
-exports.create = function (config) {
-  manageIcon(config);
-  manageSettings(config);
-
-  return AuthenticationConfiguration.create(config);
-};
+// exports.create = function (config) {
+//   manageIcon(config);
+//   manageSettings(config);
 
-exports.update = function (id, config) {
-  manageIcon(config);
-  manageSettings(config);
+//   return AuthenticationConfiguration.create(config);
+// };
 
-  return AuthenticationConfiguration.findByIdAndUpdate(id, config, { new: true }).exec();
-};
+// exports.update = function (id, config) {
+//   manageIcon(config);
+//   manageSettings(config);
 
-exports.remove = function (id) {
-  return AuthenticationConfiguration.findByIdAndRemove(id).exec();
-};
\ No newline at end of file
+//   return AuthenticationConfiguration.findByIdAndUpdate(id, config, { new: true }).exec();
+// };

From 422e6d5f0ec184a23fd350dee91bdc89419ad306 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 23 Sep 2024 17:42:05 -0600
Subject: [PATCH 109/183] fix(service): add this context to mongodb migration
 functions

---
 service/src/@types/mongodb-migrations/index.d.ts | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/service/src/@types/mongodb-migrations/index.d.ts b/service/src/@types/mongodb-migrations/index.d.ts
index 0610c1783..a078246d9 100644
--- a/service/src/@types/mongodb-migrations/index.d.ts
+++ b/service/src/@types/mongodb-migrations/index.d.ts
@@ -2,7 +2,7 @@
 
 declare module '@ngageoint/mongodb-migrations' {
 
-  import { MongoClientOptions } from 'mongodb'
+  import { MongoClientOptions, Db } from 'mongodb'
   export { MongoClientOptions } from 'mongodb'
 
   export class Migrator {
@@ -74,10 +74,15 @@ declare module '@ngageoint/mongodb-migrations' {
     options?: MongoClientOptions
   }
 
+  export type MigrationContext = {
+    db: Db
+    log: Console
+  }
+
   export interface Migration {
     id: string
-    up?: (callback: (err: any) => any) => any
-    down?: (callback: (err: any) => any) => any
+    up?: (this: MigrationContext, callback: (err: any) => any) => any
+    down?: (this: MigrationContext, callback: (err: any) => any) => any
   }
 
   export interface MigrationResult {

From 9ad2d6883dc1f5aca6c4647621f8d99db62a214b Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 25 Sep 2024 11:55:24 -0600
Subject: [PATCH 110/183] refactor(service): users/auth: progress commit: local
 signup web route and bunch of other related changes

---
 .../security/devices.security.test.ts         |  14 +-
 .../security/users.security.test.ts           |  15 +
 .../devices/adapters.devices.db.mongoose.ts   |   2 +-
 service/src/app.api/users/app.api.users.ts    |  27 ++
 .../systemInfo/app.impl.systemInfo.ts         |   1 +
 service/src/app.impl/users/app.impl.users.ts  | 124 +++++-
 service/src/app.ts                            |   1 +
 .../ingress.adapters.controllers.web.ts       | 136 ++++++
 .../ingress/ingress.adapters.db.mongoose.ts   |  32 ++
 service/src/ingress/ingress.app.api.ts        |  34 ++
 service/src/ingress/ingress.entities.ts       |  62 ++-
 .../ingress/local-idp.adapters.db.mongoose.ts | 388 +++++++++---------
 service/src/ingress/local-idp.app.api.ts      |  13 +
 service/src/ingress/local-idp.entities.ts     |  41 +-
 service/src/ingress/verification.ts           |   2 +-
 .../authentication-configuration.service.js   |   3 -
 16 files changed, 652 insertions(+), 243 deletions(-)
 create mode 100644 service/functionalTests/security/users.security.test.ts
 create mode 100644 service/src/ingress/ingress.adapters.controllers.web.ts
 create mode 100644 service/src/ingress/ingress.adapters.db.mongoose.ts
 create mode 100644 service/src/ingress/ingress.app.api.ts
 create mode 100644 service/src/ingress/local-idp.app.api.ts

diff --git a/service/functionalTests/security/devices.security.test.ts b/service/functionalTests/security/devices.security.test.ts
index c6f705a6a..dc3684d2b 100644
--- a/service/functionalTests/security/devices.security.test.ts
+++ b/service/functionalTests/security/devices.security.test.ts
@@ -1,8 +1,8 @@
 import { expect } from 'chai'
 
-describe('device operations', function() {
+describe('device management security', function() {
 
-  describe('removing', function() {
+  describe('removing a device', function() {
 
     it('invalidates associated sessions', async function() {
       expect.fail('todo')
@@ -13,9 +13,12 @@ describe('device operations', function() {
     })
   })
 
-  describe('disabling', function() {
+  /**
+   * AKA, set `registered` to `false`.
+   */
+  describe('disabling a device', function() {
 
-    it('invalidates existing associated sessions', async function() {
+    it('invalidates associated sessions', async function() {
       expect.fail('todo')
     })
 
@@ -24,6 +27,9 @@ describe('device operations', function() {
     })
   })
 
+  /**
+   * AKA, approving; set `registered` to `true`.
+   */
   describe('enabling', function() {
 
     it('allows the owning user to authenticate with the device', async function() {
diff --git a/service/functionalTests/security/users.security.test.ts b/service/functionalTests/security/users.security.test.ts
new file mode 100644
index 000000000..ab88b229d
--- /dev/null
+++ b/service/functionalTests/security/users.security.test.ts
@@ -0,0 +1,15 @@
+import { expect } from 'chai'
+
+describe('user management security', function() {
+
+  describe('disabling a user account', function() {
+
+    it('prevents the user from authenticating', async function() {
+      expect.fail('todo')
+    })
+
+    it('invalidates associated sessions', async function() {
+      expect.fail('todo')
+    })
+  })
+})
\ No newline at end of file
diff --git a/service/src/adapters/devices/adapters.devices.db.mongoose.ts b/service/src/adapters/devices/adapters.devices.db.mongoose.ts
index 8f4761dc1..1f22b01ed 100644
--- a/service/src/adapters/devices/adapters.devices.db.mongoose.ts
+++ b/service/src/adapters/devices/adapters.devices.db.mongoose.ts
@@ -5,7 +5,7 @@ import { pageOf, PageOf } from '../../entities/entities.global'
 import { BaseMongooseRepository, DocumentMapping, pageQuery } from '../base/adapters.base.db.mongoose'
 import { UserDocument } from '../users/adapters.users.db.mongoose'
 
-const Schema = mongoose.Schema;
+const Schema = mongoose.Schema
 
 export type DeviceDocument = Omit<Device, 'id' | 'userId'> & {
   _id: mongoose.Types.ObjectId
diff --git a/service/src/app.api/users/app.api.users.ts b/service/src/app.api/users/app.api.users.ts
index 1f462fd69..4b6fe9fd0 100644
--- a/service/src/app.api/users/app.api.users.ts
+++ b/service/src/app.api/users/app.api.users.ts
@@ -22,6 +22,33 @@ export type UserSearchResult = Pick<User, 'id' | 'username' | 'displayName' | 'e
 export interface SearchUsers {
   (req: UserSearchRequest): Promise<AppResponse<PageOf<UserSearchResult>, PermissionDeniedError>>
 }
+export interface ReadMyAccountRequest extends AppRequest<User> {}
+
+export interface ReadMyAccountOperation {
+  (req: ReadMyAccountRequest): Promise<AppResponse<User, PermissionDeniedError>>
+}
+
+export interface UpdateMyAccountRequest extends AppRequest<User> {}
+
+export interface UpdateMyAccountOperation {
+  (req: UpdateMyAccountRequest): Promise<AppResponse<UserExpanded, PermissionDeniedError | InvalidInputError>>
+}
+
+export interface DisableUserRequest extends AppRequest {
+  userId: UserId
+}
+
+export interface DisableUserOperation {
+  (req: DisableUserOperation): Promise<AppResponse<void, PermissionDeniedError | EntityNotFoundError>>
+}
+
+export interface RemoveUserRequest extends AppRequest {
+  userId: UserId
+}
+
+export interface RemoveUserOperation {
+  (req: RemoveUserRequest): Promise<AppResponse<UserExpanded, PermissionDeniedError | EntityNotFoundError>>
+}
 
 export interface UsersPermissionService {
   ensureReadUsersPermission(context: AppRequestContext): Promise<null | PermissionDeniedError>
diff --git a/service/src/app.impl/systemInfo/app.impl.systemInfo.ts b/service/src/app.impl/systemInfo/app.impl.systemInfo.ts
index 4054701c8..968e96b5e 100644
--- a/service/src/app.impl/systemInfo/app.impl.systemInfo.ts
+++ b/service/src/app.impl/systemInfo/app.impl.systemInfo.ts
@@ -3,6 +3,7 @@ import * as api from '../../app.api/systemInfo/app.api.systemInfo';
 import { EnvironmentService } from '../../entities/systemInfo/entities.systemInfo';
 import * as Settings from '../../models/setting';
 import * as Users from '../../models/user';
+// TODO: users-next
 import * as AuthenticationConfiguration from '../../models/authenticationconfiguration';
 import AuthenticationConfigurationTransformer from '../../transformers/authenticationconfiguration';
 import { ExoPrivilegedSystemInfo, ExoRedactedSystemInfo, ExoSystemInfo, SystemInfoPermissionService } from '../../app.api/systemInfo/app.api.systemInfo';
diff --git a/service/src/app.impl/users/app.impl.users.ts b/service/src/app.impl/users/app.impl.users.ts
index 8c8b15b9c..9ca2574bd 100644
--- a/service/src/app.impl/users/app.impl.users.ts
+++ b/service/src/app.impl/users/app.impl.users.ts
@@ -1,10 +1,114 @@
-import * as api from '../../app.api/users/app.api.users';
-import { UserRepository } from '../../entities/users/entities.users';
-import { withPermission, KnownErrorsOf } from '../../app.api/app.api.global';
-import { PageOf } from '../../entities/entities.global';
+import * as api from '../../app.api/users/app.api.users'
+import { UserRepository } from '../../entities/users/entities.users'
+import { withPermission, KnownErrorsOf } from '../../app.api/app.api.global'
+import { PageOf } from '../../entities/entities.global'
+import { IdentityProviderRepository } from '../../ingress/ingress.entities'
 
-export function SearchUsers(userRepo: UserRepository,permissions: api.UsersPermissionService
-): api.SearchUsers {
+
+export function CreateUserOperation(userRepo: UserRepository, idpRepo: IdentityProviderRepository): api.CreateUserOperation {
+  return async function createUser(req: api.CreateUserRequest): ReturnType<api.CreateUserOperation> {
+    const reqUser = req.user
+    const baseUser = {
+      ...reqUser,
+      active: false,
+      enabled: true,
+    }
+    const localIdp = await idpRepo.findIdpByName('local')
+    if (!localIdp) {
+      throw new Error('local identity provider does not exist')
+    }
+    const created = await userRepo.create(baseUser)
+    const enrollmentPolicy = localIdp.userEnrollmentPolicy
+    if (Array.isArray(enrollmentPolicy.assignToEvents) && enrollmentPolicy.assignToEvents.length > 0) {
+    }
+    if (Array.isArray(enrollmentPolicy.assignToTeams) && enrollmentPolicy.assignToTeams.length > 0) {
+    }
+
+    let defaultTeams;
+    let defaultEvents
+    if (authenticationConfig) {
+      baseUser.authentication.authenticationConfigurationId = authenticationConfig._id;
+      const requireAdminActivation = authenticationConfig.settings.usersReqAdmin || { enabled: true };
+      if (requireAdminActivation) {
+        baseUser.active = baseUser.active || !requireAdminActivation.enabled;
+      }
+
+      defaultTeams = authenticationConfig.settings.newUserTeams;
+      defaultEvents = authenticationConfig.settings.newUserEvents;
+    } else {
+      throw new Error('No configuration defined for ' + baseUser.authentication.type);
+    }
+
+    const created = await userRepo.create(baseUser)
+
+    if (options.avatar) {
+      try {
+        const avatar = avatarPath(newUser._id, newUser, options.avatar);
+        await fs.move(options.avatar.path, avatar.absolutePath);
+
+        newUser.avatar = {
+          relativePath: avatar.relativePath,
+          contentType: options.avatar.mimetype,
+          size: options.avatar.size
+        };
+
+        await newUser.save();
+      } catch { }
+    }
+
+    if (options.icon && (options.icon.type === 'create' || options.icon.type === 'upload')) {
+      try {
+        const icon = iconPath(newUser._id, newUser, options.icon);
+        await fs.move(options.icon.path, icon.absolutePath);
+
+        newUser.icon.type = options.icon.type;
+        newUser.icon.relativePath = icon.relativePath;
+        newUser.icon.contentType = options.icon.mimetype;
+        newUser.icon.size = options.icon.size;
+        newUser.icon.text = options.icon.text;
+        newUser.icon.color = options.icon.color;
+
+        await newUser.save();
+      } catch { }
+    }
+
+    if (defaultTeams && Array.isArray(defaultTeams)) {
+      const addUserToTeam = util.promisify(TeamModel.addUser);
+      for (let i = 0; i < defaultTeams.length; i++) {
+        try {
+          await addUserToTeam({ _id: defaultTeams[i] }, newUser);
+        } catch { }
+      }
+    }
+
+    if (defaultEvents && Array.isArray(defaultEvents)) {
+      const addUserToTeam = util.promisify(TeamModel.addUser);
+
+      for (let i = 0; i < defaultEvents.length; i++) {
+        const team = await TeamModel.getTeamForEvent({ _id: defaultEvents[i] });
+        if (team) {
+          try {
+            await addUserToTeam(team, newUser);
+          } catch { }
+        }
+      }
+    }
+
+    return newUser;
+  }
+}
+
+export function AdmitUserFromIdentityProvider(): api.AdminUserFromIdentityProvider {
+
+}
+
+// export function UpdateUserOperation(): api.UpdateUserOperation {
+//   return async function updateUser(req: api.UpdateUserRequest): ReturnType<api.UpdateUserOperation> {
+
+//   }
+// }
+
+export function SearchUsers(userRepo: UserRepository,permissions: api.UsersPermissionService): api.SearchUsers {
   return async function searchUsers(req: api.UserSearchRequest): ReturnType<api.SearchUsers> {
     return await withPermission<
       PageOf<api.UserSearchResult>,
@@ -25,13 +129,13 @@ export function SearchUsers(userRepo: UserRepository,permissions: api.UsersPermi
               allPhones: x.phones.reduce((allPhones, phone, index) => {
                 return index === 0
                   ? `${phone.number}`
-                  : `${allPhones}; ${phone.number}`;
+                  : `${allPhones}; ${phone.number}`
               }, '')
             };
           }
         );
-        return page;
+        return page
       }
-    );
-  };
+    )
+  }
 }
\ No newline at end of file
diff --git a/service/src/app.ts b/service/src/app.ts
index ee6c09ce9..01de1b53c 100644
--- a/service/src/app.ts
+++ b/service/src/app.ts
@@ -61,6 +61,7 @@ import { EnvironmentServiceImpl } from './adapters/systemInfo/adapters.systemInf
 import { SystemInfoAppLayer } from './app.api/systemInfo/app.api.systemInfo'
 import { CreateReadSystemInfo } from './app.impl/systemInfo/app.impl.systemInfo'
 import Settings from "./models/setting";
+// TODO: users-next
 import AuthenticationConfiguration from "./models/authenticationconfiguration";
 import AuthenticationConfigurationTransformer from "./transformers/authenticationconfiguration";
 import { SystemInfoRoutes } from './adapters/systemInfo/adapters.systemInfo.controllers.web'
diff --git a/service/src/ingress/ingress.adapters.controllers.web.ts b/service/src/ingress/ingress.adapters.controllers.web.ts
new file mode 100644
index 000000000..f1998c3c6
--- /dev/null
+++ b/service/src/ingress/ingress.adapters.controllers.web.ts
@@ -0,0 +1,136 @@
+import express from 'express'
+import svgCaptcha from 'svg-captcha'
+import { Authenticator } from 'passport'
+import { Strategy as BearerStrategy } from 'passport-http-bearer'
+import { defaultHashUtil } from '../utilities/password-hashing'
+import { JWTService, Payload, TokenVerificationError, VerificationErrorReason } from './verification'
+import { LocalIdpEnrollment } from './local-idp.entities'
+import { invalidInput, InvalidInputError, MageError } from '../app.api/app.api.errors'
+import { IdentityProviderHooks } from './ingress.entities'
+import { EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
+
+
+export type LocalIdpOperations = {
+  enrollMyself: EnrollMyselfOperation
+}
+
+export function CreateLocalIdpRoutes(localIdpApp: LocalIdpOperations, tokenService: JWTService, passport: Authenticator): express.Router {
+
+  const captchaBearer = new BearerStrategy((token, done) => {
+    const expectation = {
+      subject: null,
+      expiration: null,
+      assertion: TokenAssertion.Captcha
+    }
+    tokenService.verifyToken(token, expectation)
+      .then(payload => done(null, payload))
+      .catch(err => done(err))
+  })
+
+  const routes = express.Router()
+
+  // TODO: signup
+  // TODO: signin
+
+  // TODO: mount to /auth/local/signin
+  routes.route('/signin')
+    .post((req, res, next) => {
+
+    })
+
+  // TODO: mount to /api/users/signups
+  routes.route('/signups')
+    .post(async (req, res, next) => {
+      try {
+        const username = typeof req.body.username === 'string' ? req.body.username.trime() : ''
+        if (!username) {
+          return res.status(400).send('Invalid signup; username is required.')
+        }
+        const background = req.body.background || '#FFFFFF'
+        const captcha = svgCaptcha.create({
+          size: 6,
+          noise: 4,
+          color: false,
+          background: background.toLowerCase() !== '#ffffff' ? background : null
+        })
+        const captchaHash = await defaultHashUtil.hashPassword(captcha.text)
+        const claims = { captcha: captchaHash }
+        const verificationToken = await tokenService.generateToken(username, TokenAssertion.Captcha, 60 * 3, claims)
+        res.json({
+          token: verificationToken,
+          captcha: `data:image/svg+xml;base64,${Buffer.from(captcha.data).toString('base64')}`
+        })
+      }
+      catch (err) {
+        next(err)
+      }
+    })
+
+  routes.route('/signups/verifications')
+    .post(
+      async (req, res, next) => {
+        passport.authenticate(captchaBearer, (err: TokenVerificationError, captchaTokenPayload: Payload) => {
+          if (err) {
+            if (err.reason === VerificationErrorReason.Expired) {
+              return res.status(401).send('Captcha timeout')
+            }
+            return res.status(400).send('Invalid captcha. Please try again.')
+          }
+          if (!captchaTokenPayload) {
+            return res.status(400).send('Missing captcha token')
+          }
+          req.user = captchaTokenPayload
+          next()
+        })(req, res, next)
+      },
+      async (req, res, next) => {
+        try {
+          const isHuman = await defaultHashUtil.validPassword(req.body.captchaText, req.user.captcha)
+          if (!isHuman) {
+            return res.status(403).send('Invalid captcha. Please try again.')
+          }
+          const payload = req.user as Payload
+          const username = payload.subject!
+          const parsedEnrollment = validateEnrollment(req.body)
+          if (parsedEnrollment instanceof MageError) {
+            return next(parsedEnrollment)
+          }
+          const enrollment: EnrollMyselfRequest = {
+            ...parsedEnrollment,
+            username
+          }
+          const appRes = await localIdpApp.enrollMyself(enrollment)
+          if (appRes.success) {
+            return res.json(appRes.success)
+          }
+          next(appRes.error)
+        }
+        catch (err) {
+          next(err)
+        }
+      }
+    )
+
+  return routes
+}
+
+function validateEnrollment(input: any): Omit<EnrollMyselfRequest, 'username'> | InvalidInputError {
+  const { displayName, email, password, phone } = input
+  if (!displayName) {
+    return invalidInput('displayName is required')
+  }
+  if (!password) {
+    return invalidInput('password is required')
+  }
+  const enrollment: Omit<EnrollMyselfRequest, 'username'> = { displayName, password }
+  if (email && typeof email === 'string') {
+    if (!/^[^\s@]+@[^\s@]+\./.test(email)) {
+      return invalidInput('email is invalid')
+    }
+    enrollment.email = email
+  }
+  if (phone && typeof phone === 'string') {
+    enrollment.phone = phone
+  }
+  return enrollment
+}
\ No newline at end of file
diff --git a/service/src/ingress/ingress.adapters.db.mongoose.ts b/service/src/ingress/ingress.adapters.db.mongoose.ts
new file mode 100644
index 000000000..b5ca2860f
--- /dev/null
+++ b/service/src/ingress/ingress.adapters.db.mongoose.ts
@@ -0,0 +1,32 @@
+import mongoose from 'mongoose'
+
+type ObjectId = mongoose.Types.ObjectId
+const Schema = mongoose.Schema
+
+const UserIngressSchema = new Schema(
+  {
+    // TODO: type is really not necessary
+    type: { type: String, required: true },
+    id: { type: String, required: false },
+    authenticationConfigurationId: { type: Schema.Types.ObjectId, ref: 'AuthenticationConfiguration', required: false }
+  },
+  {
+    timestamps: {
+      updatedAt: 'lastUpdated'
+    },
+    toObject: {
+      transform: DbAuthenticationToObject
+    }
+  }
+);
+
+export type UserIdpAccountDocument = {
+  _id: ObjectId
+  // TODO: migrate to this foreign key instead of on user records
+  // userId: ObjectId
+  createdAt: Date
+  lastUpdated: Date
+  // TODO: migrate to identityProviderId
+  authenticationConfigurationId: ObjectId
+  idpAccount: Record<string, any>
+}
\ No newline at end of file
diff --git a/service/src/ingress/ingress.app.api.ts b/service/src/ingress/ingress.app.api.ts
new file mode 100644
index 000000000..4fc69ee11
--- /dev/null
+++ b/service/src/ingress/ingress.app.api.ts
@@ -0,0 +1,34 @@
+import { InvalidInputError, PermissionDeniedError } from '../app.api/app.api.errors'
+import { AppRequest, AppResponse } from '../app.api/app.api.global'
+import { Avatar, User, UserIcon } from '../entities/users/entities.users'
+
+
+export interface EnrollMyselfRequest {
+  username: string
+  password: string
+  displayName: string
+  phone?: string | null
+  email?: string | null
+}
+
+/**
+ * Create the given account in the local identity provider.
+ */
+export interface EnrollMyselfOperation {
+  (req: EnrollMyselfRequest): Promise<AppResponse<User, InvalidInputError>>
+}
+
+export interface CreateUserRequest extends AppRequest {
+  user: Omit<User, 'id' | 'icon' | 'avatar' | 'authenticationId' | 'active' | 'enabled'>
+  password: string
+  icon?: UserIcon & { content: NodeJS.ReadableStream | Buffer }
+  avatar?: Avatar & { content: NodeJS.ReadableStream | Buffer }
+}
+
+/**
+ * Manually create an account on behalf of another user using the Mage local IDP.  This is the use case of an admin
+ * creating an account for another user.
+ */
+export interface CreateUserOperation {
+  (req: CreateUserRequest): Promise<AppResponse<User, InvalidInputError | PermissionDeniedError>>
+}
diff --git a/service/src/ingress/ingress.entities.ts b/service/src/ingress/ingress.entities.ts
index d5e8103bf..1fe653016 100644
--- a/service/src/ingress/ingress.entities.ts
+++ b/service/src/ingress/ingress.entities.ts
@@ -1,3 +1,4 @@
+import { User } from '../entities/users/entities.users'
 import { Device, DeviceId } from '../entities/devices/entities.devices'
 import { MageEventId } from '../entities/events/entities.events'
 import { TeamId } from '../entities/teams/entities.teams'
@@ -36,10 +37,11 @@ export interface AuthenticationProtocol {
 export type IdentityProviderId = string
 
 /**
- * An identity provider (IDP) is a service maintains user profiles and that Mage trusts to authenticate user
- * credentials via a specific authentication protocol.  Mage delegates user authentication to identity providers.
- * Within Mage, the identity provider implementation maps the provider's user profile/account attributes to a Mage
- * user profile.
+ * An identity provider (IDP) is a service that maintains user profiles and that Mage trusts to authenticate user
+ * credentials using a specific authentication protocol.  Mage delegates user authentication to identity providers.
+ * Within Mage, the identity provider's protocol implementation maps the provider's user profile/account attributes to
+ * a Mage user profile.  This identity provider entity encapsulates the authentication protocol parameters to enable
+ * communication to a specific identity provider service.
  */
 export interface IdentityProvider {
   id: IdentityProviderId
@@ -49,31 +51,61 @@ export interface IdentityProvider {
   protocolSettings: Record<string, any>
   enabled: boolean
   lastUpdated: Date
-  enrollmentPolicy: EnrollmentPolicy
+  userEnrollmentPolicy: UserEnrollmentPolicy
   deviceEnrollmentPolicy: DeviceEnrollmentPolicy
-  description?: string | null
-  textColor?: string | null
-  buttonColor?: string | null
-  icon?: Buffer | null
+  textColor?: string
+  buttonColor?: string
+  icon?: Buffer
 }
 
 /**
  * Enrollment policy defines rules and effects to apply when a new user establishes a Mage account.
  */
-export interface EnrollmentPolicy {
+export interface UserEnrollmentPolicy {
+  /**
+   * When true, an administrator must approve and activate new user accounts.
+   */
+  accountApprovalRequired: boolean
   // TODO: configurable role assignment
   // assignRole: string
   assignToTeams: TeamId[]
   assignToEvents: MageEventId[]
-  requireAccountApproval: boolean
-
 }
 
 export interface DeviceEnrollmentPolicy {
-  requireDeviceApproval: boolean
+  /**
+   * When true, an administrator must approve and activate new devices associated with user accounts.
+   */
+  deviceApprovalRequired: boolean
+}
+
+/**
+ * The identity provider user is the result of mapping a specific IDP account to a Mage user account.
+ */
+export type IdentityProviderUser = Pick<User, 'username' | 'displayName' | 'email' | 'phones'>
+
+export interface IdentityProviderHooks {
+  /**
+   * Indicate that a user has authenticated with the given identity provider and Mage can continue enrollment and/or
+   * establish a session for the user.
+   */
+  admitUserFromIdentityProvider(account: IdentityProviderUser, idp: IdentityProvider): unknown
+  /**
+   * Indicate the given user has ended their session and logged out of the given identity provider, or the user has
+   * revoked access for Mage to use the IDP for authentication.
+   */
+  terminateSessionsForUser(username: string, idp: IdentityProvider): unknown
+  accountDisabled(username: string, idp: IdentityProvider): unknown
+  accountEnabled(username: string, idp: IdentityProvider): unknown
 }
 
 export interface IdentityProviderRepository {
-  findById(id: IdentityProviderId): Promise<IdentityProvider | null>
-  findByName(name: string): Promise<IdentityProvider | null>
+  findIdpById(id: IdentityProviderId): Promise<IdentityProvider | null>
+  findIdpByName(name: string): Promise<IdentityProvider | null>
+  /**
+   * Update the IDP according to patch semantics.  Remove keys in the given update with `undefined` values from the
+   * saved record.  Keys not present in the given update will have no affect on the saved record.
+   */
+  updateIdp(update: Partial<IdentityProvider> & Pick<IdentityProvider, 'id'>): Promise<IdentityProvider | null>
+  deleteIdp(id: IdentityProviderId): Promise<IdentityProvider | null>
 }
diff --git a/service/src/ingress/local-idp.adapters.db.mongoose.ts b/service/src/ingress/local-idp.adapters.db.mongoose.ts
index 5fcd38f2b..0ec11ef42 100644
--- a/service/src/ingress/local-idp.adapters.db.mongoose.ts
+++ b/service/src/ingress/local-idp.adapters.db.mongoose.ts
@@ -1,47 +1,27 @@
 "use strict";
 
 import mongoose from 'mongoose'
-import { LocalIdpAccount, LocalIdpRepository } from './local-idp.entities'
-
-const async = require('async')
-const hasher = require('../utilities/pbkdf2')()
-const User = require('./user')
-const Token = require('./token')
-// TODO: users-next
-const PasswordValidator = require('../utilities/passwordValidator');
-
-const Schema = mongoose.Schema;
-
-const AuthenticationSchema = new Schema(
-  {
-    // TODO: type is really not necessary
-    type: { type: String, required: true },
-    id: { type: String, required: false },
-    authenticationConfigurationId: { type: Schema.Types.ObjectId, ref: 'AuthenticationConfiguration', required: false }
-  },
-  {
-    discriminatorKey: 'type',
-    timestamps: {
-      updatedAt: 'lastUpdated'
-    },
-    toObject: {
-      transform: DbAuthenticationToObject
-    }
-  }
-);
-
-function DbAuthenticationToObject(authIn, authOut, options) {
-  delete authOut._id
-  authOut.id = authIn._id
-  if (authIn.populated('authenticationConfigurationId') && authIn.authenticationConfigurationId) {
-    delete authOut.authenticationConfigurationId;
-    authOut.authenticationConfiguration = authIn.authenticationConfigurationId.toObject(options);
-  }
-  return authOut;
+import { IdentityProviderModel } from './identity-providers.adapters.db.mongoose'
+import { DuplicateUsernameError, LocalIdpAccount, LocalIdpRepository, SecurityPolicy } from './local-idp.entities'
+
+const Schema = mongoose.Schema
+
+export type LocalIdpAccountDocument = Omit<LocalIdpAccount, 'id' | 'hashedPassword' | 'previousHashedPasswords'> & {
+  /**
+   * The _id is the username on the acccount.
+   */
+  _id: string
+  password: string
+  previousPasswords: string[]
 }
 
-const LocalSchema = new Schema(
+export type LocalIdpAccountModel = mongoose.Model<LocalIdpAccountDocument>
+
+// TODO: migrate from old authentication schema
+export const LocalIdpAccountSchema = new Schema(
   {
+    // TODO: users-next: migration to set username as _id
+    _id: { type: String, required: true },
     password: { type: String, required: true },
     previousPasswords: { type: [String], default: [] },
     security: {
@@ -52,192 +32,206 @@ const LocalSchema = new Schema(
     }
   },
   {
-    toObject: {
-      transform: DbLocalAuthenticationToObject
+    id: false,
+    timestamps: {
+      createdAt: true,
+      updatedAt: 'lastUpdated'
     }
   }
-);
+)
 
-function DbLocalAuthenticationToObject(authIn, authOut, options) {
-  authOut = DbAuthenticationToObject(authIn, authOut, options)
-  delete authOut.password;
-  delete authOut.previousPasswords;
-  return authOut;
-}
+// function DbLocalAuthenticationToObject(authIn, authOut, options) {
+//   authOut = DbAuthenticationToObject(authIn, authOut, options)
+//   delete authOut.password;
+//   delete authOut.previousPasswords;
+//   return authOut;
+// }
 
-const SamlSchema = new Schema({});
-const LdapSchema = new Schema({});
-const OauthSchema = new Schema({});
-const OpenIdConnectSchema = new Schema({});
+// const SamlSchema = new Schema({});
+// const LdapSchema = new Schema({});
+// const OauthSchema = new Schema({});
+// const OpenIdConnectSchema = new Schema({});
 
-AuthenticationSchema.method('validatePassword', function (password, callback) {
-  hasher.validPassword(password, this.password, callback);
-});
+// AuthenticationSchema.method('validatePassword', function (password, callback) {
+//   hasher.validPassword(password, this.password, callback);
+// });
 
 // Encrypt password before save
-LocalSchema.pre('save', function (next) {
-  const authentication = this;
-
-  // only hash the password if it has been modified (or is new)
-  if (!authentication.isModified('password')) {
-    return next();
-  }
-
-  async.waterfall([
-    function (done) {
-      AuthenticationConfiguration.getById(authentication.authenticationConfigurationId).then(localConfiguration => {
-        done(null, localConfiguration.settings.passwordPolicy);
-      }).catch(err => done(err));
-    },
-    function (policy, done) {
-      const { password, previousPasswords } = authentication;
-      PasswordValidator.validate(policy, { password, previousPasswords }).then(validationStatus => {
-        if (!validationStatus.valid) {
-          const err = new Error(validationStatus.errorMsg);
-          err.status = 400;
-          return done(err);
-        }
-
-        done(null, policy);
-      });
-    },
-    function (policy, done) {
-      hasher.hashPassword(authentication.password, function (err, password) {
-        done(err, policy, password);
-      });
-    }
-  ], function (err, policy, password) {
-    if (err) return next(err);
-
-    authentication.password = password;
-    authentication.previousPasswords.unshift(password);
-    authentication.previousPasswords = authentication.previousPasswords.slice(0, policy.passwordHistoryCount);
-    next();
-  });
-});
+// LocalSchema.pre('save', function (next) {
+//   const authentication = this;
+
+//   // only hash the password if it has been modified (or is new)
+//   if (!authentication.isModified('password')) {
+//     return next();
+//   }
+
+//   async.waterfall([
+//     function (done) {
+//       AuthenticationConfiguration.getById(authentication.authenticationConfigurationId).then(localConfiguration => {
+//         done(null, localConfiguration.settings.passwordPolicy);
+//       }).catch(err => done(err));
+//     },
+//     function (policy, done) {
+//       const { password, previousPasswords } = authentication;
+//       PasswordValidator.validate(policy, { password, previousPasswords }).then(validationStatus => {
+//         if (!validationStatus.valid) {
+//           const err = new Error(validationStatus.errorMsg);
+//           err.status = 400;
+//           return done(err);
+//         }
+
+//         done(null, policy);
+//       });
+//     },
+//     function (policy, done) {
+//       hasher.hashPassword(authentication.password, function (err, password) {
+//         done(err, policy, password);
+//       });
+//     }
+//   ], function (err, policy, password) {
+//     if (err) return next(err);
+
+//     authentication.password = password;
+//     authentication.previousPasswords.unshift(password);
+//     authentication.previousPasswords = authentication.previousPasswords.slice(0, policy.passwordHistoryCount);
+//     next();
+//   });
+// });
 
 // Remove Token if password changed
-LocalSchema.pre('save', function (next) {
-  const authentication = this;
-
-  // only remove token if password has been modified (or is new)
-  if (!authentication.isModified('password')) {
-    return next();
+// LocalSchema.pre('save', function (next) {
+//   const authentication = this;
+
+//   // only remove token if password has been modified (or is new)
+//   if (!authentication.isModified('password')) {
+//     return next();
+//   }
+
+//   async.waterfall([
+//     function (done) {
+//       // TODO: users-next
+//       User.getUserByAuthenticationId(authentication._id, function (err, user) {
+//         done(err, user);
+//       });
+//     },
+//     function (user, done) {
+//       if (user) {
+//         Token.removeTokensForUser(user, function (err) {
+//           done(err);
+//         });
+//       } else {
+//         done();
+//       }
+//     }
+//   ], function (err) {
+//     return next(err);
+//   });
+// });
+
+// exports.getAuthenticationByStrategy = function (strategy, uid, callback) {
+//   if (callback) {
+//     Authentication.findOne({ id: uid, type: strategy }, callback);
+//   } else {
+//     return Authentication.findOne({ id: uid, type: strategy });
+//   }
+// };
+
+// exports.getAuthenticationsByType = function (type) {
+//   return Authentication.find({ type: type }).exec();
+// };
+
+// exports.getAuthenticationsByAuthConfigId = function (authConfigId) {
+//   return Authentication.find({ authenticationConfigurationId: authConfigId }).exec();
+// };
+
+// exports.countAuthenticationsByAuthConfigId = function (authConfigId) {
+//   return Authentication.count({ authenticationConfigurationId: authConfigId }).exec();
+// };
+
+// exports.createAuthentication = function (authentication) {
+//   const document = {
+//     id: authentication.id,
+//     type: authentication.type,
+//     authenticationConfigurationId: authentication.authenticationConfigurationId,
+//   }
+
+//   if (authentication.type === 'local') {
+//     document.password = authentication.password;
+//     document.security ={
+//       lockedUntil: null
+//     }
+//   }
+
+//   return Authentication.create(document);
+// };
+
+// exports.updateAuthentication = function (authentication) {
+//   return authentication.save();
+// };
+
+// exports.removeAuthenticationById = function (authenticationId, done) {
+//   Authentication.findByIdAndRemove(authenticationId, done);
+// };
+
+function entityForDocument(doc: LocalIdpAccountDocument): LocalIdpAccount {
+  return {
+    username: doc._id,
+    createdAt: doc.createdAt,
+    lastUpdated: doc.lastUpdated,
+    hashedPassword: doc.password,
+    previousHashedPasswords: [ ...doc.previousPasswords ],
+    security: { ...doc.security }
   }
+}
 
-  async.waterfall([
-    function (done) {
-      // TODO: users-next
-      User.getUserByAuthenticationId(authentication._id, function (err, user) {
-        done(err, user);
-      });
-    },
-    function (user, done) {
-      if (user) {
-        Token.removeTokensForUser(user, function (err) {
-          done(err);
-        });
-      } else {
-        done();
-      }
-    }
-  ], function (err) {
-    return next(err);
-  });
-});
-
-AuthenticationSchema.virtual('authenticationConfiguration').get(function () {
-  return this.populated('authenticationConfigurationId') ? this.authenticationConfigurationId : null;
-});
-
-const Authentication = mongoose.model('Authentication', AuthenticationSchema);
-exports.Model = Authentication;
-
-const LocalAuthentication = Authentication.discriminator('local', LocalSchema);
-exports.Local = LocalAuthentication;
-
-const SamlAuthentication = Authentication.discriminator('saml', SamlSchema);
-exports.SAML = SamlAuthentication;
-
-const LdapAuthentication = Authentication.discriminator('ldap', LdapSchema);
-exports.LDAP = LdapAuthentication;
-
-const OauthAuthentication = Authentication.discriminator('oauth', OauthSchema);
-exports.Oauth = OauthAuthentication;
-
-const OpenIdConnectAuthentication = Authentication.discriminator('openidconnect', OpenIdConnectSchema);
-exports.OpenIdConnect = OpenIdConnectAuthentication;
-
-exports.getAuthenticationByStrategy = function (strategy, uid, callback) {
-  if (callback) {
-    Authentication.findOne({ id: uid, type: strategy }, callback);
-  } else {
-    return Authentication.findOne({ id: uid, type: strategy });
+// TODO: verify desired behavior for this mapping
+function documentForEntity(entity: Partial<LocalIdpAccount>): Partial<LocalIdpAccountDocument> {
+  const hasKey = (key: keyof LocalIdpAccount): boolean => Object.prototype.hasOwnProperty.call(entity, key)
+  const doc: Partial<LocalIdpAccountDocument> = {}
+  if (hasKey('username')) {
+    doc._id = entity.username
   }
-};
-
-exports.getAuthenticationsByType = function (type) {
-  return Authentication.find({ type: type }).exec();
-};
-
-exports.getAuthenticationsByAuthConfigId = function (authConfigId) {
-  return Authentication.find({ authenticationConfigurationId: authConfigId }).exec();
-};
-
-exports.countAuthenticationsByAuthConfigId = function (authConfigId) {
-  return Authentication.count({ authenticationConfigurationId: authConfigId }).exec();
-};
-
-exports.createAuthentication = function (authentication) {
-  const document = {
-    id: authentication.id,
-    type: authentication.type,
-    authenticationConfigurationId: authentication.authenticationConfigurationId,
+  if (hasKey('createdAt') && entity.createdAt) {
+    doc.createdAt = new Date(entity.createdAt)
   }
-
-  if (authentication.type === 'local') {
-    document.password = authentication.password;
-    document.security ={
-      lockedUntil: null
-    }
+  if (hasKey('lastUpdated') && entity.lastUpdated) {
+    doc.lastUpdated = new Date(entity.lastUpdated)
   }
-
-  return Authentication.create(document);
-};
-
-exports.updateAuthentication = function (authentication) {
-  return authentication.save();
-};
-
-exports.removeAuthenticationById = function (authenticationId, done) {
-  Authentication.findByIdAndRemove(authenticationId, done);
-};
+  if (hasKey('hashedPassword')) {
+    doc.password = entity.hashedPassword
+  }
+  if (hasKey('previousHashedPasswords') && entity.previousHashedPasswords) {
+    doc.previousPasswords = entity.previousHashedPasswords
+  }
+  if (hasKey('security') && entity.security) {
+    doc.security = { ...entity.security }
+  }
+  return doc
+}
 
 export class LocalIdpMongooseRepository implements LocalIdpRepository {
 
-  constructor(private UserModel: UserModel, private AuthenticationModel: AuthenticationModel) {
+  constructor(private LocalIdpAccountModel: LocalIdpAccountModel) {}
 
+  async createLocalAccount(account: LocalIdpAccount): Promise<LocalIdpAccount | DuplicateUsernameError> {
+    const doc = documentForEntity(account)
+    const created = await this.LocalIdpAccountModel.create(doc)
+    return entityForDocument(created)
   }
 
-  createLocalAccount(account: LocalIdpAccount): Promise<LocalIdpAccount> {
-    throw new Error('Method not implemented.')
-  }
-
-  async readLocalAccount(id: LocalIdpAccountId): Promise<LocalIdpAccount | null> {
-    const dbId = new mongoose.Types.ObjectId(id)
-    const userModelInstance = await this.UserModel.findOne({ username })
-      .populate({ path: 'authenticationId', populate: 'authenticationConfigurationId' })
-    if (!userModelInstance) {
-      return null
+  async readLocalAccount(username: string): Promise<LocalIdpAccount | null> {
+    const doc = await this.LocalIdpAccountModel.findById(username, null, { lean: true })
+    if (doc) {
+      return entityForDocument(doc)
     }
-    return userModelInstance.authenticationId
+    return null
   }
 
   updateLocalAccount(update: Partial<LocalIdpAccount> & Pick<LocalIdpAccount, 'username'>): Promise<LocalIdpAccount | null> {
     throw new Error('Method not implemented.')
   }
 
-  deleteLocalAccount(id: string): Promise<LocalIdpAccount | null> {
+  deleteLocalAccount(username: string): Promise<LocalIdpAccount | null> {
     throw new Error('Method not implemented.')
   }
 }
\ No newline at end of file
diff --git a/service/src/ingress/local-idp.app.api.ts b/service/src/ingress/local-idp.app.api.ts
new file mode 100644
index 000000000..8a0420ded
--- /dev/null
+++ b/service/src/ingress/local-idp.app.api.ts
@@ -0,0 +1,13 @@
+import { EntityNotFoundError, InvalidInputError } from '../app.api/app.api.errors'
+import { AppRequest, AppResponse } from '../app.api/app.api.global'
+import { LocalIdpAccount } from './local-idp.entities'
+
+
+export interface LocalIdpAuthenticateRequest extends AppRequest {
+  username: string
+  password: string
+}
+
+export interface LocalIdpAuthenticateOperation {
+  (req: LocalIdpAuthenticateRequest): Promise<AppResponse<LocalIdpAccount, EntityNotFoundError | InvalidInputError>>
+}
\ No newline at end of file
diff --git a/service/src/ingress/local-idp.entities.ts b/service/src/ingress/local-idp.entities.ts
index 4c134057b..801d9abda 100644
--- a/service/src/ingress/local-idp.entities.ts
+++ b/service/src/ingress/local-idp.entities.ts
@@ -1,9 +1,10 @@
 import { PasswordRequirements } from '../utilities/password-policy'
-
-export type LocalIdpAccountId = string
+import { IdentityProvider } from './ingress.entities'
 
 export interface LocalIdpAccount {
-  id: LocalIdpAccountId
+  username: string
+  createdAt: Date
+  lastUpdated: Date
   hashedPassword: string
   previousHashedPasswords: string[]
   security: {
@@ -17,9 +18,6 @@ export interface LocalIdpAccount {
 export interface LocalIdpEnrollment {
   username: string
   password: string
-  displayName: string
-  email?: string
-  phone?: string
 }
 
 export interface AccountLockPolicy {
@@ -44,10 +42,29 @@ export interface SecurityPolicy {
 }
 
 export interface LocalIdpRepository {
-  readSecurityPolicy(): Promise<SecurityPolicy>
-  updateSecurityPolicy(policy: SecurityPolicy): Promise<SecurityPolicy>
-  createLocalAccount(account: LocalIdpAccount): Promise<LocalIdpAccount>
-  readLocalAccount(id: LocalIdpAccountId): Promise<LocalIdpAccount | null>
-  updateLocalAccount(update: Partial<LocalIdpAccount> & Pick<LocalIdpAccount, 'id'>): Promise<LocalIdpAccount | null>
-  deleteLocalAccount(id: LocalIdpAccountId): Promise<LocalIdpAccount | null>
+  // readSecurityPolicy(): Promise<SecurityPolicy>
+  // updateSecurityPolicy(policy: SecurityPolicy): Promise<SecurityPolicy>
+  createLocalAccount(account: LocalIdpAccount): Promise<LocalIdpAccount | DuplicateUsernameError>
+  readLocalAccount(username: string): Promise<LocalIdpAccount | null>
+  updateLocalAccount(update: Partial<LocalIdpAccount> & Pick<LocalIdpAccount, 'username'>): Promise<LocalIdpAccount | null>
+  deleteLocalAccount(username: string): Promise<LocalIdpAccount | null>
+}
+
+export function localIdpSecurityPolicyFromIdenityProvider(localIdp: IdentityProvider): SecurityPolicy {
+  const settings = localIdp.protocolSettings
+  return {
+    accountLock: { ...settings.accountLock },
+    passwordRequirements: { ...settings.passwordPolicy }
+  }
+}
+
+export class LocalIdpError extends Error {
+
+}
+
+export class DuplicateUsernameError extends LocalIdpError {
+
+  constructor(public username: string) {
+    super(`duplicate account username: ${username}`)
+  }
 }
\ No newline at end of file
diff --git a/service/src/ingress/verification.ts b/service/src/ingress/verification.ts
index 63175a786..3010b269f 100644
--- a/service/src/ingress/verification.ts
+++ b/service/src/ingress/verification.ts
@@ -30,7 +30,7 @@ export class TokenGenerateError extends Error {
   }
 }
 
-class TokenVerificationError extends Error {
+export class TokenVerificationError extends Error {
 
   /**
    * @param reason why the verification failed
diff --git a/web-app/src/ng1/factories/authentication-configuration.service.js b/web-app/src/ng1/factories/authentication-configuration.service.js
index b915ee65d..878798c86 100644
--- a/web-app/src/ng1/factories/authentication-configuration.service.js
+++ b/web-app/src/ng1/factories/authentication-configuration.service.js
@@ -6,9 +6,6 @@ function AuthenticationConfigurationService($http, $httpParamSerializer) {
         return $http.get('/api/authentication/configuration/', { params: options });
     }
 
-    /**
-     * TODO: why is this using form encoding instead of straight json?
-     */
     function updateConfiguration(config) {
         return $http.put('/api/authentication/configuration/' + config._id, config, {
             headers: {

From 1dd2c22f62ba232fc915a1ad771cf43007cc6776 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 25 Sep 2024 12:29:39 -0600
Subject: [PATCH 111/183] refactor(service): users/auth: no need to extend app
 request for local idp auth request

---
 service/src/ingress/local-idp.app.api.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/service/src/ingress/local-idp.app.api.ts b/service/src/ingress/local-idp.app.api.ts
index 8a0420ded..4603f33cb 100644
--- a/service/src/ingress/local-idp.app.api.ts
+++ b/service/src/ingress/local-idp.app.api.ts
@@ -1,9 +1,9 @@
 import { EntityNotFoundError, InvalidInputError } from '../app.api/app.api.errors'
-import { AppRequest, AppResponse } from '../app.api/app.api.global'
+import { AppResponse } from '../app.api/app.api.global'
 import { LocalIdpAccount } from './local-idp.entities'
 
 
-export interface LocalIdpAuthenticateRequest extends AppRequest {
+export interface LocalIdpAuthenticateRequest {
   username: string
   password: string
 }

From 5480013eded89aef96a5cf3000c326635695a10d Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 1 Oct 2024 23:50:08 -0600
Subject: [PATCH 112/183] refactor(service): users/auth: local auth protocol
 mostly implemented

---
 service/src/ingress/ingress.protocol.local.ts | 136 ++++++------------
 .../ingress/local-idp.adapters.db.mongoose.ts |  24 +++-
 service/src/ingress/local-idp.app.api.ts      |  13 +-
 service/src/ingress/local-idp.app.impl.ts     |  27 ++++
 service/src/ingress/local-idp.entities.ts     | 125 ++++++++++++++--
 service/src/utilities/password-policy.ts      |  14 +-
 6 files changed, 219 insertions(+), 120 deletions(-)
 create mode 100644 service/src/ingress/local-idp.app.impl.ts

diff --git a/service/src/ingress/ingress.protocol.local.ts b/service/src/ingress/ingress.protocol.local.ts
index 0a70175fb..fb9163b18 100644
--- a/service/src/ingress/ingress.protocol.local.ts
+++ b/service/src/ingress/ingress.protocol.local.ts
@@ -1,96 +1,52 @@
-const log = require('winston')
-  , moment = require('moment')
-  , LocalStrategy = require('passport-local').Strategy
-  , TokenAssertion = require('./verification').TokenAssertion
-  , User = require('../models/user')
-  , userTransformer = require('../transformers/user')
-  , { app, passport, tokenService } = require('./index')
-  , Authentication = require('../models/authentication');
-
-function configure() {
-  log.info('Configuring local authentication');
-  passport.use(new LocalStrategy(
-    function (username, password, done) {
-      // TODO: users-next
-      User.getUserByUsername(username, function (err, user) {
-        if (err) { return done(err); }
-
-        if (!user) {
-          log.warn('Failed login attempt: User with username ' + username + ' not found');
-          return done(null, false, { message: 'Please check your username and password and try again.' });
-        }
-
-        if (!user.active) {
-          log.warn('Failed user login attempt: User ' + user.username + ' is not active');
-          return done(null, false, { message: 'User account is not approved, please contact your MAGE administrator to approve your account.' });
-        }
-
-        if (!user.enabled) {
-          log.warn('Failed user login attempt: User ' + user.username + ' account is disabled.');
-          return done(null, false, { message: 'Your account has been disabled, please contact a MAGE administrator for assistance.' });
-        }
-
-        const settings = user.authentication.security;
-        if (settings && settings.locked && moment().isBefore(moment(settings.lockedUntil))) {
-          log.warn('Failed user login attempt: User ' + user.username + ' account is locked until ' + settings.lockedUntil);
-          return done(null, false, { message: 'Your account has been temporarily locked, please try again later or contact a MAGE administrator for assistance.' });
-        }
-
-        if (!(user.authentication instanceof Authentication.Local)) {
-          log.warn(user.username + " is not a local account");
-          return done(null, false, { message: 'You do not have a local account, please contact a MAGE administrator for assistance.' });
-        }
-
-        if (!user.authentication.authenticationConfiguration.enabled) {
-          log.warn(user.authentication.authenticationConfiguration.title + " authentication is not enabled");
-          return done(null, false, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-        }
-
-        user.authentication.validatePassword(password, function (err, isValid) {
-          if (err) return done(err);
+import passport from 'passport'
+import express from 'express'
+import { Strategy as LocalStrategy, VerifyFunction as LocalStrategyVerifyFunction } from 'passport-local'
+import { LocalIdpAccount } from './local-idp.entities'
+import { IdentityProviderUser } from './ingress.entities'
+import { LocalIdpAuthenticateOperation } from './local-idp.app.api'
+
+
+function userForLocalIdpAccount(account: LocalIdpAccount): IdentityProviderUser {
+  return {
+    username: account.username,
+    displayName: account.username,
+    phones: [],
+  }
+}
 
-          if (isValid) {
-            // TODO: users-next
-            User.validLogin(user)
-              .then(() => done(null, user))
-              .catch(err => done(err));
-          } else {
-            log.warn('Failed login attempt: User with username ' + username + ' provided an invalid password');
-            // TODO: users-next
-            User.invalidLogin(user)
-              .then(() => done(null, false, { message: 'Please check your username and password and try again.' }))
-              .catch(err => done(err));
-          }
-        });
-      });
+function createProtocolMiddleware(localIdpAuthenticate: LocalIdpAuthenticateOperation): passport.Strategy {
+  const verify: LocalStrategyVerifyFunction = async function LocalIngressProtocolVerify(username, password, done) {
+    const authResult = await localIdpAuthenticate({ username, password })
+    if (authResult.success) {
+      const localAccount = authResult.success
+      const localIdpUser = userForLocalIdpAccount(localAccount)
+      return done(null, { from: 'identityProvider', account: localIdpUser })
     }
-  ));
+    return done(authResult.error)
+  }
+  return new LocalStrategy(verify)
 }
 
-function initialize() {
-  configure();
-
-  app.post('/auth/local/signin',
-    function authenticate(req, res, next) {
-      passport.authenticate('local', function (err, user, info = {}) {
-        if (err) return next(err);
-
-        if (!user) return res.status(401).send(info.message);
-
-        tokenService.generateToken(user._id.toString(), TokenAssertion.Authorized, 60 * 5)
-          .then(token => {
-            res.json({
-              token: token,
-              user: userTransformer.transform(user, { path: req.getRoot() })
-            });
-          }).catch(err => {
-            next(err);
-          });
-      })(req, res, next);
-    }
-  );
+const validateRequest: express.RequestHandler = function LocalProtocolIngressHandler(req, res, next) {
+  if (req.method !== 'POST' || !req.body) {
+    return res.status(400).send(`invalid request method ${req.method}`)
+  }
+  const username = req.body.username
+  const password = req.body.password
+  if (typeof username !== 'string' || typeof password !== 'string') {
+    return res.status(400).send(`username and password are required`)
+  }
+  next()
 }
 
-module.exports = {
-  initialize
-}
\ No newline at end of file
+export function createIngressStack(passport: passport.Authenticator, localIdpAuthenticate: LocalIdpAuthenticateOperation): express.RequestHandler[] {
+  const authStrategy = createProtocolMiddleware(localIdpAuthenticate)
+  const authHandler: express.RequestHandler = async (req, res, next) => {
+    passport.authenticate(authStrategy)(req, res, next)
+  }
+  return [
+    express.urlencoded(),
+    validateRequest,
+    authHandler,
+  ]
+}
diff --git a/service/src/ingress/local-idp.adapters.db.mongoose.ts b/service/src/ingress/local-idp.adapters.db.mongoose.ts
index 0ec11ef42..228b10286 100644
--- a/service/src/ingress/local-idp.adapters.db.mongoose.ts
+++ b/service/src/ingress/local-idp.adapters.db.mongoose.ts
@@ -1,8 +1,8 @@
 "use strict";
 
 import mongoose from 'mongoose'
-import { IdentityProviderModel } from './identity-providers.adapters.db.mongoose'
-import { DuplicateUsernameError, LocalIdpAccount, LocalIdpRepository, SecurityPolicy } from './local-idp.entities'
+import { IdentityProviderDocument, IdentityProviderModel } from './identity-providers.adapters.db.mongoose'
+import { LocalIdpDuplicateUsernameError, LocalIdpAccount, LocalIdpRepository, SecurityPolicy, localIdpSecurityPolicyFromIdenityProvider } from './local-idp.entities'
 
 const Schema = mongoose.Schema
 
@@ -209,11 +209,27 @@ function documentForEntity(entity: Partial<LocalIdpAccount>): Partial<LocalIdpAc
   return doc
 }
 
+function localIdpSecurityPolicyFromIdenityProvider(localIdp: IdentityProviderDocument): SecurityPolicy {
+  const settings = localIdp.settings
+  return {
+    accountLock: { ...settings.accountLock },
+    passwordRequirements: { ...settings.passwordPolicy }
+  }
+}
+
 export class LocalIdpMongooseRepository implements LocalIdpRepository {
 
-  constructor(private LocalIdpAccountModel: LocalIdpAccountModel) {}
+  constructor(private LocalIdpAccountModel: LocalIdpAccountModel, private IdentityProviderModel: IdentityProviderModel) {}
+
+  async readSecurityPolicy(): Promise<SecurityPolicy> {
+    const idpDoc = await this.IdentityProviderModel.findOne({ name: 'local' })
+    if (idpDoc) {
+      return localIdpSecurityPolicyFromIdenityProvider(idpDoc)
+    }
+    throw new Error('local identity provider not found')
+  }
 
-  async createLocalAccount(account: LocalIdpAccount): Promise<LocalIdpAccount | DuplicateUsernameError> {
+  async createLocalAccount(account: LocalIdpAccount): Promise<LocalIdpAccount | LocalIdpDuplicateUsernameError> {
     const doc = documentForEntity(account)
     const created = await this.LocalIdpAccountModel.create(doc)
     return entityForDocument(created)
diff --git a/service/src/ingress/local-idp.app.api.ts b/service/src/ingress/local-idp.app.api.ts
index 4603f33cb..8c170695f 100644
--- a/service/src/ingress/local-idp.app.api.ts
+++ b/service/src/ingress/local-idp.app.api.ts
@@ -1,13 +1,12 @@
 import { EntityNotFoundError, InvalidInputError } from '../app.api/app.api.errors'
 import { AppResponse } from '../app.api/app.api.global'
-import { LocalIdpAccount } from './local-idp.entities'
+import { LocalIdpAccount, LocalIdpCredentials } from './local-idp.entities'
 
 
-export interface LocalIdpAuthenticateRequest {
-  username: string
-  password: string
+export interface LocalIdpAuthenticateOperation {
+  (req: LocalIdpCredentials): Promise<AppResponse<LocalIdpAccount, EntityNotFoundError | InvalidInputError>>
 }
 
-export interface LocalIdpAuthenticateOperation {
-  (req: LocalIdpAuthenticateRequest): Promise<AppResponse<LocalIdpAccount, EntityNotFoundError | InvalidInputError>>
-}
\ No newline at end of file
+export interface LocalIdpCreateAccountOperation {
+  (req: LocalIdpCredentials): Promise<AppResponse<LocalIdpAccount, EntityNotFoundError | InvalidInputError>>
+}
diff --git a/service/src/ingress/local-idp.app.impl.ts b/service/src/ingress/local-idp.app.impl.ts
new file mode 100644
index 000000000..3e55fe846
--- /dev/null
+++ b/service/src/ingress/local-idp.app.impl.ts
@@ -0,0 +1,27 @@
+import { invalidInput } from '../app.api/app.api.errors'
+import { AppResponse } from '../app.api/app.api.global'
+import { LocalIdpAuthenticateOperation } from './local-idp.app.api'
+import { attemptAuthentication, LocalIdpRepository } from './local-idp.entities'
+
+
+export function CreateLocalIdpAuthenticateOperation(repo: LocalIdpRepository): LocalIdpAuthenticateOperation {
+  return async function localIdpAuthenticate(req): ReturnType<LocalIdpAuthenticateOperation> {
+    const account = await repo.readLocalAccount(req.username)
+    if (!account) {
+      console.info('local account does not exist:', req.username)
+      return AppResponse.error(invalidInput(`Failed to authenticate user ${req.username}`))
+    }
+    const securityPolicy = await repo.readSecurityPolicy()
+    const attempt = await attemptAuthentication(account, req.password, securityPolicy.accountLock)
+    if (attempt.failed) {
+      console.info('local authentication failed', attempt.failed)
+      return AppResponse.error(invalidInput(`Failed to authenticate user ${req.username}`))
+    }
+    const accountSaved = await repo.updateLocalAccount(attempt.authenticated)
+    if (accountSaved) {
+      return AppResponse.success(accountSaved)
+    }
+    console.error(`account for username ${req.username} did not exist for update after authentication`)
+    return AppResponse.error(invalidInput(`Failed to authenticate user ${req.username}`))
+  }
+}
\ No newline at end of file
diff --git a/service/src/ingress/local-idp.entities.ts b/service/src/ingress/local-idp.entities.ts
index 801d9abda..d7a75217b 100644
--- a/service/src/ingress/local-idp.entities.ts
+++ b/service/src/ingress/local-idp.entities.ts
@@ -1,5 +1,6 @@
-import { PasswordRequirements } from '../utilities/password-policy'
-import { IdentityProvider } from './ingress.entities'
+import moment from 'moment'
+import { PasswordRequirements, validatePasswordRequirements } from '../utilities/password-policy'
+import { defaultHashUtil } from '../utilities/password-hashing'
 
 export interface LocalIdpAccount {
   username: string
@@ -9,13 +10,16 @@ export interface LocalIdpAccount {
   previousHashedPasswords: string[]
   security: {
     locked: boolean
-    lockedUntil: Date
+    /**
+     * When `lockedUntil` is `null`, the account is locked indefinitely.
+     */
+    lockedUntil: Date | null
     invalidLoginAttempts: number
     numberOfTimesLocked: number
   }
 }
 
-export interface LocalIdpEnrollment {
+export interface LocalIdpCredentials {
   username: string
   password: string
 }
@@ -42,29 +46,126 @@ export interface SecurityPolicy {
 }
 
 export interface LocalIdpRepository {
-  // readSecurityPolicy(): Promise<SecurityPolicy>
+  readSecurityPolicy(): Promise<SecurityPolicy>
   // updateSecurityPolicy(policy: SecurityPolicy): Promise<SecurityPolicy>
-  createLocalAccount(account: LocalIdpAccount): Promise<LocalIdpAccount | DuplicateUsernameError>
+  createLocalAccount(account: LocalIdpAccount): Promise<LocalIdpAccount | LocalIdpDuplicateUsernameError>
   readLocalAccount(username: string): Promise<LocalIdpAccount | null>
   updateLocalAccount(update: Partial<LocalIdpAccount> & Pick<LocalIdpAccount, 'username'>): Promise<LocalIdpAccount | null>
   deleteLocalAccount(username: string): Promise<LocalIdpAccount | null>
 }
 
-export function localIdpSecurityPolicyFromIdenityProvider(localIdp: IdentityProvider): SecurityPolicy {
-  const settings = localIdp.protocolSettings
+export async function prepareNewAccount(username: string, password: string, policy: SecurityPolicy): Promise<LocalIdpAccount | LocalIdpInvalidPasswordError> {
+  const passwordRequirements = policy.passwordRequirements
+  const passwordValidation = await validatePasswordRequirements(password, passwordRequirements, [])
+  if (!passwordValidation.valid) {
+    return invalidPasswordError(passwordValidation.errorMessage || 'Password does not meet requirements.')
+  }
+  const hashedPassword = await defaultHashUtil.hashPassword(password)
+  const now = new Date()
+  const account: LocalIdpAccount = {
+    username,
+    hashedPassword: defaultHashUtil.serializePassword(hashedPassword),
+    previousHashedPasswords: [],
+    createdAt: now,
+    lastUpdated: now,
+    security: {
+      invalidLoginAttempts: 0,
+      locked: false,
+      lockedUntil: null,
+      numberOfTimesLocked: 0
+    }
+  }
+  return account
+}
+
+export function verifyPasswordForAccount(account: LocalIdpAccount, password: string): Promise<boolean> {
+  return defaultHashUtil.validPassword(password, account.hashedPassword)
+}
+
+export function applyPolicyForFailedAuthenticationAttempt(account: LocalIdpAccount, policy: AccountLockPolicy): LocalIdpAccount {
+  if (!policy.enabled) {
+    return account
+  }
+  const accountStatus = { ...account.security }
+  accountStatus.invalidLoginAttempts += 1
+  if (accountStatus.invalidLoginAttempts >= policy.lockAfterInvalidLoginCount) {
+    accountStatus.locked = true
+    accountStatus.numberOfTimesLocked += 1
+    if (accountStatus.numberOfTimesLocked >= policy.disableAfterLockCount) {
+      accountStatus.lockedUntil = null
+    }
+    else {
+      accountStatus.lockedUntil = moment().add(policy.lockDurationSeconds, 'seconds').toDate()
+    }
+  }
+  else {
+    accountStatus.locked = false
+    accountStatus.lockedUntil = null
+  }
   return {
-    accountLock: { ...settings.accountLock },
-    passwordRequirements: { ...settings.passwordPolicy }
+    ...account,
+    security: accountStatus
   }
 }
 
-export class LocalIdpError extends Error {
+export type LocalIdpAuthenticationResult = { authenticated: LocalIdpAccount, failed: false } | { authenticated: false, failed: LocalIdpFailedAuthenticationError }
 
+/**
+ * Check whether the given password matches the given account's password.  If the password does not match, or the
+ * account is locked, return an error whose account object reflects the given account lock policy.  If the password
+ * matches, return the given account with {@link unlockAndResetSecurityStatus() good security status}.
+ */
+export async function attemptAuthentication(account: LocalIdpAccount, password: string, policy: AccountLockPolicy): Promise<LocalIdpAuthenticationResult> {
+  if (account.security.locked) {
+    if (account.security.lockedUntil === null || account.security.lockedUntil.getTime() > Date.now()) {
+      return { authenticated: false, failed: accountLockedError(account) }
+    }
+  }
+  const passwordMatches = await verifyPasswordForAccount(account, password)
+  if (passwordMatches) {
+    return { authenticated: unlockAndResetSecurityStatusOfAccount(account), failed: false }
+  }
+  return { authenticated: false, failed: passwordMismatchError(applyPolicyForFailedAuthenticationAttempt(account, policy)) }
 }
 
-export class DuplicateUsernameError extends LocalIdpError {
+export function unlockAndResetSecurityStatusOfAccount(account: LocalIdpAccount): LocalIdpAccount {
+  const accountStatus = { ...account.security }
+  accountStatus.locked = false
+  accountStatus.lockedUntil = null
+  accountStatus.invalidLoginAttempts = 0
+  accountStatus.numberOfTimesLocked = 0
+  return {
+    ...account,
+    security: accountStatus
+  }
+}
 
+export class LocalIdpError extends Error {
+}
+
+export class LocalIdpInvalidPasswordError extends LocalIdpError {
+}
+
+export class LocalIdpDuplicateUsernameError extends LocalIdpError {
   constructor(public username: string) {
     super(`duplicate account username: ${username}`)
   }
+}
+
+export class LocalIdpFailedAuthenticationError extends LocalIdpError {
+  constructor(public account: LocalIdpAccount, message: string = `failed to authenticate user ${account.username}`) {
+    super(message)
+  }
+}
+
+function invalidPasswordError(reason: string): LocalIdpError {
+  return new LocalIdpError(reason)
+}
+
+function passwordMismatchError(account: LocalIdpAccount): LocalIdpFailedAuthenticationError {
+  return new LocalIdpFailedAuthenticationError(account, `invalid password for user ${account.username}`)
+}
+
+function accountLockedError(account: LocalIdpAccount): LocalIdpFailedAuthenticationError {
+  return new LocalIdpFailedAuthenticationError(account, `account for user ${account.username} is locked${account.security.lockedUntil ? account.security.lockedUntil.toUTCString() : ''}`)
 }
\ No newline at end of file
diff --git a/service/src/utilities/password-policy.ts b/service/src/utilities/password-policy.ts
index d6e4ea55b..421db79be 100644
--- a/service/src/utilities/password-policy.ts
+++ b/service/src/utilities/password-policy.ts
@@ -27,14 +27,14 @@ export type PasswordRequirements = {
 
 export type PasswordValidationResult = {
   valid: boolean
-  errorMsg: string | null
+  errorMessage: string | null
 }
 
-export async function validate(policy: PasswordRequirements, { password, previousPasswords }: { password: string, previousPasswords: string[] }): Promise<PasswordValidationResult> {
+export async function validatePasswordRequirements(password: string, policy: PasswordRequirements, previousPasswords: string[]): Promise<PasswordValidationResult> {
   if (!password) {
     return {
       valid: false,
-      errorMsg: 'Password is missing'
+      errorMessage: 'Password is missing'
     }
   }
   const valid =
@@ -46,7 +46,7 @@ export async function validate(policy: PasswordRequirements, { password, previou
     validateMinimumNumbers(policy, password) &&
     validateMinimumSpecialCharacters(policy, password) &&
     (await validatePasswordHistory(policy, password, previousPasswords))
-  return { valid, errorMsg: valid ? null : policy.helpText }
+  return { valid, errorMessage: valid ? null : policy.helpText }
 }
 
 function validatePasswordLength(policy: PasswordRequirements, password: string): boolean {
@@ -151,11 +151,11 @@ function createRestrictedRegex(restrictedChars: string): string {
   return forbiddenRegex
 }
 
-async function validatePasswordHistory(policy: PasswordRequirements, password: string, previousPasswords: string[]): Promise<boolean> {
-  if (!policy.passwordHistoryCountEnabled || !previousPasswords) {
+async function validatePasswordHistory(policy: PasswordRequirements, password: string, previousPasswordHashes: string[]): Promise<boolean> {
+  if (!policy.passwordHistoryCountEnabled || !previousPasswordHashes) {
     return true
   }
-  const truncatedHistory = previousPasswords.slice(0, policy.passwordHistoryCount)
+  const truncatedHistory = previousPasswordHashes.slice(0, policy.passwordHistoryCount)
   for (const previousPasswordHash of truncatedHistory) {
     const used = await defaultHashUtil.validPassword(password, previousPasswordHash)
     if (used) {

From d4af596f2878b6cf8385ca8943c95de726acb894 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 2 Oct 2024 09:42:45 -0600
Subject: [PATCH 113/183] refactor(service): users/auth: implement create local
 idp account operation

---
 service/src/ingress/local-idp.app.impl.ts | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/service/src/ingress/local-idp.app.impl.ts b/service/src/ingress/local-idp.app.impl.ts
index 3e55fe846..c35807235 100644
--- a/service/src/ingress/local-idp.app.impl.ts
+++ b/service/src/ingress/local-idp.app.impl.ts
@@ -1,7 +1,7 @@
 import { invalidInput } from '../app.api/app.api.errors'
 import { AppResponse } from '../app.api/app.api.global'
-import { LocalIdpAuthenticateOperation } from './local-idp.app.api'
-import { attemptAuthentication, LocalIdpRepository } from './local-idp.entities'
+import { LocalIdpAuthenticateOperation, LocalIdpCreateAccountOperation } from './local-idp.app.api'
+import { attemptAuthentication, LocalIdpDuplicateUsernameError, LocalIdpError, LocalIdpInvalidPasswordError, LocalIdpRepository, prepareNewAccount } from './local-idp.entities'
 
 
 export function CreateLocalIdpAuthenticateOperation(repo: LocalIdpRepository): LocalIdpAuthenticateOperation {
@@ -24,4 +24,22 @@ export function CreateLocalIdpAuthenticateOperation(repo: LocalIdpRepository): L
     console.error(`account for username ${req.username} did not exist for update after authentication`)
     return AppResponse.error(invalidInput(`Failed to authenticate user ${req.username}`))
   }
+}
+
+export function CreateLocalIdpCreateAccountOperation(repo: LocalIdpRepository): LocalIdpCreateAccountOperation {
+  return async function localIdpCreateAccount(req) {
+    const securityPolicy = await repo.readSecurityPolicy()
+    const candidateAccount = await prepareNewAccount(req.username, req.password, securityPolicy)
+    if (candidateAccount instanceof LocalIdpInvalidPasswordError) {
+      return AppResponse.error(invalidInput(`Failed to create account ${req.username}.`, [ candidateAccount.message, 'password' ]))
+    }
+    const createdAccount = await repo.createLocalAccount(candidateAccount)
+    if (createdAccount instanceof LocalIdpError) {
+      if (createdAccount instanceof LocalIdpDuplicateUsernameError) {
+        console.info(`attempted to create local account with duplicate username ${req.username}`)
+      }
+      return AppResponse.error(invalidInput(`Failed to create account ${req.username}.`))
+    }
+    return AppResponse.success(createdAccount)
+  }
 }
\ No newline at end of file

From 09888d713d16287d19dade4dfb627422334676a8 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 2 Oct 2024 10:49:36 -0600
Subject: [PATCH 114/183] refactor(service): users/auth: remove unnecessary
 error from create local account operation

---
 service/src/ingress/local-idp.app.api.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/ingress/local-idp.app.api.ts b/service/src/ingress/local-idp.app.api.ts
index 8c170695f..70ad88862 100644
--- a/service/src/ingress/local-idp.app.api.ts
+++ b/service/src/ingress/local-idp.app.api.ts
@@ -8,5 +8,5 @@ export interface LocalIdpAuthenticateOperation {
 }
 
 export interface LocalIdpCreateAccountOperation {
-  (req: LocalIdpCredentials): Promise<AppResponse<LocalIdpAccount, EntityNotFoundError | InvalidInputError>>
+  (req: LocalIdpCredentials): Promise<AppResponse<LocalIdpAccount, InvalidInputError>>
 }

From 8ae412b99017543c3c638111a213f25554a5ff2b Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 10 Oct 2024 14:44:33 -0600
Subject: [PATCH 115/183] refactor(service): users/auth: align identity
 provider document to entity with intent to create a database migration

---
 ...identity-providers.adapters.db.mongoose.ts | 69 ++++++++-----------
 .../ingress/local-idp.adapters.db.mongoose.ts |  3 +-
 2 files changed, 32 insertions(+), 40 deletions(-)

diff --git a/service/src/ingress/identity-providers.adapters.db.mongoose.ts b/service/src/ingress/identity-providers.adapters.db.mongoose.ts
index f5f9a19c7..1be0fae37 100644
--- a/service/src/ingress/identity-providers.adapters.db.mongoose.ts
+++ b/service/src/ingress/identity-providers.adapters.db.mongoose.ts
@@ -15,20 +15,8 @@ export type CommonIdpSettings = {
   newUserEvents?: MageEventId[]
 }
 
-export type IdentityProviderDocument = {
+export type IdentityProviderDocument = Omit<IdentityProvider, 'id'> & {
   _id: ObjectId
-  name: string
-  /**
-   * IDP type maps to an ingress authentication protocol.
-   */
-  type: string
-  lastUpdated: Date
-  enabled: boolean
-  title?: string
-  settings: CommonIdpSettings & Record<string, any>
-  textColor?: string
-  buttonColor?: string
-  icon?: Buffer
 }
 
 export type IdentityProviderModel = mongoose.Model<IdentityProviderDocument>
@@ -36,13 +24,22 @@ export type IdentityProviderModel = mongoose.Model<IdentityProviderDocument>
 export const IdentityProviderSchema = new Schema<IdentityProviderDocument>(
   {
     name: { type: String, required: true },
-    type: { type: String, required: true },
+    protocol: { type: String, required: true },
+    protocolSettings: Schema.Types.Mixed,
+    userEnrollmentPolicy: {
+      accountApprovalRequired: { type: Boolean, required: true, default: true },
+      assignRole: { type: String, required: true },
+      assignToEvents: { type: [Number], default: [] },
+      assignToTeams: { type: [String], default: [] },
+    },
+    deviceEnrollmentPolicy: {
+      deviceApprovalRequired: { type: Boolean, required: true, default: true },
+    },
     title: { type: String, required: false },
     textColor: { type: String, required: false },
     buttonColor: { type: String, required: false },
     icon: { type: Buffer, required: false },
     enabled: { type: Boolean, default: true },
-    settings: Schema.Types.Mixed
   },
   {
     timestamps: {
@@ -55,24 +52,16 @@ export const IdentityProviderSchema = new Schema<IdentityProviderDocument>(
 IdentityProviderSchema.index({ name: 1, type: 1 }, { unique: true })
 
 export function idpEntityForDocument(doc: IdentityProviderDocument): IdentityProvider {
-  const settings = doc.settings || {}
-  const userEnrollmentPolicy: UserEnrollmentPolicy = {
-    accountApprovalRequired: !!settings.usersReqAdmin?.enabled,
-    assignToTeams: doc.settings.newUserTeams || [],
-    assignToEvents: doc.settings.newUserEvents || [],
-  }
-  const deviceEnrollmentPolicy: DeviceEnrollmentPolicy = {
-    deviceApprovalRequired: !!settings.devicesReqAdmin?.enabled
-  }
+  const userEnrollmentPolicy: UserEnrollmentPolicy = { ...doc.userEnrollmentPolicy }
+  const deviceEnrollmentPolicy: DeviceEnrollmentPolicy = { ...doc.deviceEnrollmentPolicy }
   return {
     id: doc._id.toHexString(),
     name: doc.name,
     enabled: doc.enabled,
     lastUpdated: doc.lastUpdated,
-    // TODO: use protocol instance if appropriate
-    protocol: { name: doc.type },
+    protocol: doc.protocol,
     title: doc.title || doc.name,
-    protocolSettings: doc.settings,
+    protocolSettings: { ...doc.protocolSettings },
     textColor: doc.textColor,
     buttonColor: doc.buttonColor,
     icon: doc.icon,
@@ -82,22 +71,24 @@ export function idpEntityForDocument(doc: IdentityProviderDocument): IdentityPro
 }
 
 export function idpDocumentForEntity(entity: Partial<IdentityProvider>): Partial<IdentityProviderDocument> {
-  // TODO: maybe delegate to protocol to copy settings
-  const settings = entity.protocolSettings ? { ...entity.protocolSettings } as CommonIdpSettings : undefined
-  const { userEnrollmentPolicy, deviceEnrollmentPolicy } = entity
-  if (settings && userEnrollmentPolicy) {
-    settings.usersReqAdmin = { enabled: !!userEnrollmentPolicy?.accountApprovalRequired }
-    settings.newUserTeams = userEnrollmentPolicy?.assignToTeams || []
-    settings.newUserEvents = userEnrollmentPolicy?.assignToEvents || []
-  }
-  if (settings && deviceEnrollmentPolicy) {
-    settings.devicesReqAdmin = { enabled: !!deviceEnrollmentPolicy?.deviceApprovalRequired }
-  }
   const doc = {} as Partial<IdentityProviderDocument>
   const entityHasKey = (key: keyof IdentityProvider): boolean => Object.prototype.hasOwnProperty.call(entity, key)
   if (entityHasKey('id')) {
     doc._id = new mongoose.Types.ObjectId(entity.id)
   }
+  if (entityHasKey('protocol')) {
+    doc.protocol = entity.protocol
+  }
+  if (entityHasKey('protocolSettings')) {
+    // TODO: maybe delegate to protocol to copy settings
+    doc.protocolSettings = { ...entity.protocolSettings }
+  }
+  if (entityHasKey('userEnrollmentPolicy')) {
+    doc.userEnrollmentPolicy = { ...entity.userEnrollmentPolicy! }
+  }
+  if (entityHasKey('deviceEnrollmentPolicy')) {
+    doc.deviceEnrollmentPolicy = { ...entity.deviceEnrollmentPolicy! }
+  }
   if (entityHasKey('buttonColor')) {
     doc.buttonColor = entity.buttonColor
   }
@@ -114,7 +105,7 @@ export function idpDocumentForEntity(entity: Partial<IdentityProvider>): Partial
     doc.name = entity.name
   }
   if (entityHasKey('protocol')) {
-    doc.type = entity.protocol?.name
+    doc.protocol = entity.protocol
   }
   if (entityHasKey('textColor')) {
     doc.textColor = entity.textColor
diff --git a/service/src/ingress/local-idp.adapters.db.mongoose.ts b/service/src/ingress/local-idp.adapters.db.mongoose.ts
index 228b10286..8134ca8f3 100644
--- a/service/src/ingress/local-idp.adapters.db.mongoose.ts
+++ b/service/src/ingress/local-idp.adapters.db.mongoose.ts
@@ -210,7 +210,7 @@ function documentForEntity(entity: Partial<LocalIdpAccount>): Partial<LocalIdpAc
 }
 
 function localIdpSecurityPolicyFromIdenityProvider(localIdp: IdentityProviderDocument): SecurityPolicy {
-  const settings = localIdp.settings
+  const settings = localIdp.protocolSettings
   return {
     accountLock: { ...settings.accountLock },
     passwordRequirements: { ...settings.passwordPolicy }
@@ -232,6 +232,7 @@ export class LocalIdpMongooseRepository implements LocalIdpRepository {
   async createLocalAccount(account: LocalIdpAccount): Promise<LocalIdpAccount | LocalIdpDuplicateUsernameError> {
     const doc = documentForEntity(account)
     const created = await this.LocalIdpAccountModel.create(doc)
+    // TODO: handle duplicate username error
     return entityForDocument(created)
   }
 

From 679db00e9274b57d673a402a3fb87742243e29ae Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 15 Oct 2024 17:09:09 -0600
Subject: [PATCH 116/183] refactor(service): users/auth: add function to create
 an enrollment candidate user entity

---
 service/src/ingress/ingress.entities.ts | 32 ++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/service/src/ingress/ingress.entities.ts b/service/src/ingress/ingress.entities.ts
index 1fe653016..ed97bd5a7 100644
--- a/service/src/ingress/ingress.entities.ts
+++ b/service/src/ingress/ingress.entities.ts
@@ -3,6 +3,7 @@ import { Device, DeviceId } from '../entities/devices/entities.devices'
 import { MageEventId } from '../entities/events/entities.events'
 import { TeamId } from '../entities/teams/entities.teams'
 import { UserExpanded, UserId } from '../entities/users/entities.users'
+import { RoleId } from '../entities/authorization/entities.authorization'
 
 export interface Session {
   token: string
@@ -30,9 +31,7 @@ export interface SessionRepository {
  * that the user is valid.  Authentication protocols are OpenID Connect, OAuth, SAML, LDAP, and Mage's own local
  * password authentication database.
  */
-export interface AuthenticationProtocol {
-  name: string
-}
+export type AuthenticationProtocolId = 'local' | 'ldap' | 'oauth' | 'oidc' | 'saml'
 
 export type IdentityProviderId = string
 
@@ -47,7 +46,7 @@ export interface IdentityProvider {
   id: IdentityProviderId
   name: string
   title: string
-  protocol: AuthenticationProtocol
+  protocol: AuthenticationProtocolId
   protocolSettings: Record<string, any>
   enabled: boolean
   lastUpdated: Date
@@ -66,8 +65,7 @@ export interface UserEnrollmentPolicy {
    * When true, an administrator must approve and activate new user accounts.
    */
   accountApprovalRequired: boolean
-  // TODO: configurable role assignment
-  // assignRole: string
+  assignRole: RoleId
   assignToTeams: TeamId[]
   assignToEvents: MageEventId[]
 }
@@ -109,3 +107,25 @@ export interface IdentityProviderRepository {
   updateIdp(update: Partial<IdentityProvider> & Pick<IdentityProvider, 'id'>): Promise<IdentityProvider | null>
   deleteIdp(id: IdentityProviderId): Promise<IdentityProvider | null>
 }
+
+/**
+ * Return a new user object from the given identity provider account information suitable to persist as newly enrolled
+ * user.  The enrollment policy for the identity provider determines the `active` flag and assigned role for the new
+ * user.
+ */
+export function createEnrollmentCandidateUser(idpAccount: IdentityProviderUser, idp: IdentityProvider): Omit<User, 'id'> {
+  const policy = idp.userEnrollmentPolicy
+  const now = new Date()
+  const candidate: Omit<User, 'id'> = {
+    active: !policy.accountApprovalRequired,
+    roleId: policy.assignRole,
+    enabled: true,
+    createdAt: now,
+    lastUpdated: now,
+    avatar: {},
+    icon: {},
+    recentEventIds: [],
+    ...idpAccount
+  }
+  return candidate
+}

From 6107f7c99d661b0a4322848c76c43584cbbb9333 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 22 Oct 2024 22:41:51 -0600
Subject: [PATCH 117/183] refactor(service): users/auth: ensure user account
 idp bindings on admission

---
 service/src/ingress/ingress.app.api.ts  |  35 +++++-
 service/src/ingress/ingress.app.impl.ts | 146 ++++++++++++++++++++++++
 service/src/ingress/ingress.entities.ts |  33 +++---
 3 files changed, 197 insertions(+), 17 deletions(-)
 create mode 100644 service/src/ingress/ingress.app.impl.ts

diff --git a/service/src/ingress/ingress.app.api.ts b/service/src/ingress/ingress.app.api.ts
index 4fc69ee11..a663bfc36 100644
--- a/service/src/ingress/ingress.app.api.ts
+++ b/service/src/ingress/ingress.app.api.ts
@@ -1,6 +1,7 @@
-import { InvalidInputError, PermissionDeniedError } from '../app.api/app.api.errors'
+import { EntityNotFoundError, InfrastructureError, InvalidInputError, MageError, PermissionDeniedError } from '../app.api/app.api.errors'
 import { AppRequest, AppResponse } from '../app.api/app.api.global'
-import { Avatar, User, UserIcon } from '../entities/users/entities.users'
+import { Avatar, User, UserExpanded, UserIcon } from '../entities/users/entities.users'
+import { IdentityProviderUser } from './ingress.entities'
 
 
 export interface EnrollMyselfRequest {
@@ -12,10 +13,10 @@ export interface EnrollMyselfRequest {
 }
 
 /**
- * Create the given account in the local identity provider.
+ * Create the given account for the requesting user in the local identity provider.
  */
 export interface EnrollMyselfOperation {
-  (req: EnrollMyselfRequest): Promise<AppResponse<User, InvalidInputError>>
+  (req: EnrollMyselfRequest): Promise<AppResponse<UserExpanded, InvalidInputError>>
 }
 
 export interface CreateUserRequest extends AppRequest {
@@ -32,3 +33,29 @@ export interface CreateUserRequest extends AppRequest {
 export interface CreateUserOperation {
   (req: CreateUserRequest): Promise<AppResponse<User, InvalidInputError | PermissionDeniedError>>
 }
+
+export interface AdmitFromIdentityProviderRequest {
+  identityProviderName: string
+  identityProviderUser: IdentityProviderUser
+}
+
+export interface AdmitFromIdentityProviderResult {
+  mageAccount: User
+  admissionToken: string
+}
+
+/**
+ * Admit a user that has authenticated with a configured identity provider.  User admission includes implicit user
+ * enrollment, i.e., when the Mage account does not exist, create a new Mage account for the user bound to the
+ * authenticating identity provider, and apply the enrollment policy configured for the identity provider.
+ */
+export interface AdmitFromIdentityProviderOperation {
+  (req: AdmitFromIdentityProviderRequest): Promise<AppResponse<AdmitFromIdentityProviderResult, EntityNotFoundError | AuthenticationFailedError | InfrastructureError>>
+}
+
+export const ErrAuthenticationFailed = Symbol.for('MageError.Ingress.AuthenticationFailed')
+export type AuthenticationFailedErrorData = { username: string, identityProviderName: string }
+export type AuthenticationFailedError = MageError<typeof ErrAuthenticationFailed, AuthenticationFailedErrorData>
+export function authenticationFailedError(username: string, identityProviderName: string, message?: string): AuthenticationFailedError {
+  return new MageError(ErrAuthenticationFailed, { username, identityProviderName }, message || `Authentication failed: ${username} @(${identityProviderName})`)
+}
\ No newline at end of file
diff --git a/service/src/ingress/ingress.app.impl.ts b/service/src/ingress/ingress.app.impl.ts
new file mode 100644
index 000000000..32ebcf0d1
--- /dev/null
+++ b/service/src/ingress/ingress.app.impl.ts
@@ -0,0 +1,146 @@
+import { entityNotFound, infrastructureError } from '../app.api/app.api.errors'
+import { AppResponse } from '../app.api/app.api.global'
+import { MageEventId } from '../entities/events/entities.events'
+import { Team, TeamId } from '../entities/teams/entities.teams'
+import { User, UserId, UserRepository, UserRepositoryError } from '../entities/users/entities.users'
+import { AdmitFromIdentityProviderOperation, AdmitFromIdentityProviderRequest, authenticationFailedError, EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
+import { createEnrollmentCandidateUser, IdentityProvider, IdentityProviderRepository, IdentityProviderUser, UserIngressBinding, UserIngressBindingRepository, UserIngressBindings } from './ingress.entities'
+import { LocalIdpCreateAccountOperation } from './local-idp.app.api'
+import { JWTService, TokenAssertion } from './verification'
+
+
+export function CreateEnrollMyselfOperation(createLocalIdpAccount: LocalIdpCreateAccountOperation, idpRepo: IdentityProviderRepository, userRepo: UserRepository): EnrollMyselfOperation {
+  return async function enrollMyself(req: EnrollMyselfRequest): ReturnType<EnrollMyselfOperation> {
+    const localAccountCreate = await createLocalIdpAccount(req)
+    if (localAccountCreate.error) {
+      return AppResponse.error(localAccountCreate.error)
+    }
+    const localAccount = localAccountCreate.success!
+    const candidateMageAccount: Partial<User> = {
+      username: localAccount.username,
+      displayName: req.displayName,
+    }
+    if (req.email) {
+      candidateMageAccount.email = req.email
+    }
+    if (req.phone) {
+      candidateMageAccount.phones = [ { number: req.phone, type: 'Main' } ]
+    }
+    const localIdp = await idpRepo.findIdpByName('local')
+    // TODO: auto-activate account after enrollment policy
+    throw new Error('unimplemented')
+  }
+}
+
+export interface AssignTeamMember {
+  (member: UserId, team: TeamId): Promise<boolean>
+}
+
+export interface FindEventTeam {
+  (mageEventId: MageEventId): Promise<Team | null>
+}
+
+type TeamAssignmentResult = { teamId: TeamId, assigned: boolean }
+type EventAssignmentResult = { eventId: MageEventId, teamId: TeamId | null, assigned: boolean }
+type EnrollmentTeamAssignmentResult = {
+  teamAssignments: TeamAssignmentResult[]
+  eventAssignments: EventAssignmentResult[]
+}
+
+async function enrollNewUser(idpAccount: IdentityProviderUser, idp: IdentityProvider, userRepo: UserRepository, ingressBindingRepo: UserIngressBindingRepository, findEventTeam: FindEventTeam, assignTeamMember: AssignTeamMember): Promise<{ mageAccount: User, ingressBindings: UserIngressBindings }> {
+  console.info(`enrolling new user account ${idpAccount.username} from identity provider ${idp.name}`)
+  const candidate = createEnrollmentCandidateUser(idpAccount, idp)
+  const mageAccount = await userRepo.create(candidate)
+  if (mageAccount instanceof UserRepositoryError) {
+    throw mageAccount
+  }
+  const ingressBindings = await ingressBindingRepo.saveUserIngressBinding(
+    mageAccount.id,
+    {
+      userId: mageAccount.id,
+      idpId: idp.id,
+      idpAccountId: idpAccount.username,
+      idpAccountAttrs: {},
+      // TODO: these do not have functionality yet
+      verified: true,
+      enabled: true,
+    }
+  )
+  if (ingressBindings instanceof Error) {
+    throw ingressBindings
+  }
+  const { assignToTeams, assignToEvents } = idp.userEnrollmentPolicy
+  const assignEnrolledToTeam = (teamId: TeamId): Promise<{ teamId: TeamId, assigned: boolean }> => {
+    return assignTeamMember(mageAccount.id, teamId)
+      .then(assigned => ({ teamId, assigned }))
+      .catch(err => {
+        console.error(`error assigning enrolled user ${mageAccount.username} to team ${teamId}`, err)
+        return { teamId, assigned: false }
+      })
+  }
+  const assignEnrolledToEventTeam = (eventId: MageEventId): Promise<{ eventId: MageEventId, teamId: TeamId | null, assigned: boolean }> => {
+    return findEventTeam(eventId)
+      .then<{ eventId: MageEventId, teamId: TeamId | null, assigned: boolean }>(eventTeam => {
+        if (eventTeam) {
+          return assignEnrolledToTeam(eventTeam.id).then(teamAssignment => ({ eventId, ...teamAssignment }))
+        }
+        console.error(`failed to find implicit team for event ${eventId} while enrolling user ${mageAccount.username}`)
+        return { eventId, teamId: null, assigned: false }
+      })
+      .catch(err => {
+        console.error(`error looking up implicit team for event ${eventId} while enrolling user ${mageAccount.username}`, err)
+        return { eventId, teamId: null, assigned: false }
+      })
+  }
+  await Promise.all([ ...assignToTeams.map(assignEnrolledToTeam), ...assignToEvents.map(assignEnrolledToEventTeam) ])
+  return { mageAccount, ingressBindings }
+}
+
+export function CreateAdmitFromIdentityProviderOperation(idpRepo: IdentityProviderRepository, ingressBindingRepo: UserIngressBindingRepository, userRepo: UserRepository, findEventTeam: FindEventTeam, assignTeamMember: AssignTeamMember, tokenService: JWTService): AdmitFromIdentityProviderOperation {
+  return async function admitFromIdentityProvider(req: AdmitFromIdentityProviderRequest): ReturnType<AdmitFromIdentityProviderOperation> {
+    const idp = await idpRepo.findIdpByName(req.identityProviderName)
+    if (!idp) {
+      return AppResponse.error(entityNotFound(req.identityProviderName, 'IdentityProvider', `identity provider not found: ${req.identityProviderName}`))
+    }
+    const idpAccount = req.identityProviderUser
+    console.info(`admitting user ${idpAccount.username} from identity provider ${idp.name}`)
+    const mageAccount = await userRepo.findByUsername(idpAccount.username)
+      .then(existingAccount => {
+        if (existingAccount) {
+          return ingressBindingRepo.readBindingsForUser(existingAccount.id).then(ingressBindings => {
+            return { mageAccount: existingAccount, ingressBindings }
+          })
+        }
+        return enrollNewUser(idpAccount, idp, userRepo, ingressBindingRepo, findEventTeam, assignTeamMember)
+      })
+      .then(enrolled => {
+        const { mageAccount, ingressBindings } = enrolled
+        if (ingressBindings.has(idp.id)) {
+          return mageAccount
+        }
+        console.error(`user ${mageAccount.username} has no ingress binding to identity provider ${idp.name}`)
+        return null
+      })
+      .catch(err => {
+        console.error(`error creating user account ${idpAccount.username} from identity provider ${idp.name}`, err)
+        return null
+      })
+    if (!mageAccount) {
+      return AppResponse.error(authenticationFailedError(idpAccount.username, idp.name))
+    }
+    if (!mageAccount.active) {
+      return AppResponse.error(authenticationFailedError(mageAccount.username, idp.name, 'Your account requires approval from a Mage administrator.'))
+    }
+    if (!mageAccount.enabled) {
+      return AppResponse.error(authenticationFailedError(mageAccount.username, idp.name, 'Your account is disabled.'))
+    }
+    try {
+      const admissionToken = await tokenService.generateToken(mageAccount.id, TokenAssertion.Authenticated, 5 * 60)
+      return AppResponse.success({ mageAccount, admissionToken })
+    }
+    catch (err) {
+      console.error(`error generating admission token while authenticating user ${mageAccount.username}`, err)
+      return AppResponse.error(infrastructureError('An unexpected error occurred while generating an authentication token.'))
+    }
+  }
+}
diff --git a/service/src/ingress/ingress.entities.ts b/service/src/ingress/ingress.entities.ts
index ed97bd5a7..21cba76db 100644
--- a/service/src/ingress/ingress.entities.ts
+++ b/service/src/ingress/ingress.entities.ts
@@ -82,19 +82,18 @@ export interface DeviceEnrollmentPolicy {
  */
 export type IdentityProviderUser = Pick<User, 'username' | 'displayName' | 'email' | 'phones'>
 
-export interface IdentityProviderHooks {
-  /**
-   * Indicate that a user has authenticated with the given identity provider and Mage can continue enrollment and/or
-   * establish a session for the user.
-   */
-  admitUserFromIdentityProvider(account: IdentityProviderUser, idp: IdentityProvider): unknown
-  /**
-   * Indicate the given user has ended their session and logged out of the given identity provider, or the user has
-   * revoked access for Mage to use the IDP for authentication.
-   */
-  terminateSessionsForUser(username: string, idp: IdentityProvider): unknown
-  accountDisabled(username: string, idp: IdentityProvider): unknown
-  accountEnabled(username: string, idp: IdentityProvider): unknown
+/**
+ * A user ingress binding is the bridge between a Mage user and an identity provider account.  When a user attempts
+ * to authenticate to Mage through an identity provider, a binding must exist between the Mage user account and the
+ * identity provider for Mage to map the identity provider account to the Mage user account.
+ */
+export interface UserIngressBinding {
+  userId: UserId
+  idpId: IdentityProviderId
+  verified: boolean
+  enabled: boolean
+  idpAccountId?: string
+  idpAccountAttrs?: Record<string, any>
 }
 
 export interface IdentityProviderRepository {
@@ -108,6 +107,14 @@ export interface IdentityProviderRepository {
   deleteIdp(id: IdentityProviderId): Promise<IdentityProvider | null>
 }
 
+export type UserIngressBindings = Map<IdentityProviderId, UserIngressBinding>
+
+export interface UserIngressBindingRepository {
+  readBindingsForUser(userId: UserId): Promise<UserIngressBindings>
+  saveUserIngressBinding(userId: UserId, binding: UserIngressBinding): Promise<UserIngressBindings | Error>
+  deleteBinding(userId: UserId, idpId: IdentityProviderId): Promise<UserIngressBinding | null>
+}
+
 /**
  * Return a new user object from the given identity provider account information suitable to persist as newly enrolled
  * user.  The enrollment policy for the identity provider determines the `active` flag and assigned role for the new

From 94fccfc0f9b2ac5eb3f4bd0c1b3234b3dbed9cfc Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 22 Oct 2024 22:44:02 -0600
Subject: [PATCH 118/183] refactor(service): users/auth: progress on ingress
 web controller

---
 .../ingress.adapters.controllers.web.ts       | 72 +++++++++++++++----
 .../src/ingress/ingress.protocol.bindings.ts  | 12 ++++
 2 files changed, 70 insertions(+), 14 deletions(-)
 create mode 100644 service/src/ingress/ingress.protocol.bindings.ts

diff --git a/service/src/ingress/ingress.adapters.controllers.web.ts b/service/src/ingress/ingress.adapters.controllers.web.ts
index f1998c3c6..d817c864c 100644
--- a/service/src/ingress/ingress.adapters.controllers.web.ts
+++ b/service/src/ingress/ingress.adapters.controllers.web.ts
@@ -4,17 +4,27 @@ import { Authenticator } from 'passport'
 import { Strategy as BearerStrategy } from 'passport-http-bearer'
 import { defaultHashUtil } from '../utilities/password-hashing'
 import { JWTService, Payload, TokenVerificationError, VerificationErrorReason } from './verification'
-import { LocalIdpEnrollment } from './local-idp.entities'
 import { invalidInput, InvalidInputError, MageError } from '../app.api/app.api.errors'
-import { IdentityProviderHooks } from './ingress.entities'
-import { EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
+import { IdentityProviderRepository } from './ingress.entities'
+import { AdmitFromIdentityProviderOperation, EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
+import { IngressProtocolWebBinding } from './ingress.protocol.bindings'
 
+declare module 'express-serve-static-core' {
+  interface Request {
+    identityProviderService?: IngressProtocolWebBinding
+  }
+}
 
-export type LocalIdpOperations = {
+export type IngressOperations = {
   enrollMyself: EnrollMyselfOperation
+  admitFromIdentityProvider: AdmitFromIdentityProviderOperation
 }
 
-export function CreateLocalIdpRoutes(localIdpApp: LocalIdpOperations, tokenService: JWTService, passport: Authenticator): express.Router {
+function bindingFor(idpName: string): IngressProtocolWebBinding {
+  throw new Error('unimplemented')
+}
+
+export function CreateIngressRoutes(ingressApp: IngressOperations, idpRepo: IdentityProviderRepository, tokenService: JWTService, passport: Authenticator): express.Router {
 
   const captchaBearer = new BearerStrategy((token, done) => {
     const expectation = {
@@ -32,26 +42,60 @@ export function CreateLocalIdpRoutes(localIdpApp: LocalIdpOperations, tokenServi
   // TODO: signup
   // TODO: signin
 
-  // TODO: mount to /auth/local/signin
-  routes.route('/signin')
-    .post((req, res, next) => {
+  const routeToIdp = express.Router().all('/',
+    ((req, res, next) => {
+      const idpService = req.identityProviderService!
+      idpService.handleRequest(req, res, next)
+    }) as express.RequestHandler,
+    (async (err, req, res, next) => {
+      if (err) {
+        console.error('identity provider authentication error:', err)
+        return res.status(500).send('unexpected authentication result')
+      }
+      if (req.user?.from !== 'identityProvider') {
+        console.error('unexpected authentication user type:', req.user?.from)
+        return res.status(500).send('unexpected authentication result')
+      }
+      const identityProviderName = req.identityProviderService!.idp.name
+      const identityProviderUser = req.user.account
+      const ingressResult = await ingressApp.admitFromIdentityProvider({ identityProviderName, identityProviderUser })
+      if (ingressResult.error) {
+        next(ingressResult.error)
+      }
+      // if user active and enabled, send authenticated JWT and proceed to verification
+      // else
+      const account = ingressResult.success!
+
+    }) as express.ErrorRequestHandler
+  )
 
-    })
+  routes.use('/:identityProviderName',
+    (req, res, next) => {
+      const idpName = req.params.identityProviderName
+      const idpService = bindingFor(idpName)
+      if (idpService) {
+        req.identityProviderService = idpService
+        return next()
+      }
+      res.status(404).send(`${idpName} not found`)
+    },
+    routeToIdp
+  )
 
   // TODO: mount to /api/users/signups
   routes.route('/signups')
     .post(async (req, res, next) => {
       try {
-        const username = typeof req.body.username === 'string' ? req.body.username.trime() : ''
+        const username = typeof req.body.username === 'string' ? req.body.username.trim() : null
         if (!username) {
-          return res.status(400).send('Invalid signup; username is required.')
+          return res.status(400).send('Invalid signup - username is required.')
         }
-        const background = req.body.background || '#FFFFFF'
+        const background = typeof req.body.background === 'string' ? req.body.background.toLowerCase() : '#ffffff'
         const captcha = svgCaptcha.create({
           size: 6,
           noise: 4,
           color: false,
-          background: background.toLowerCase() !== '#ffffff' ? background : null
+          background: background !== '#ffffff' ? background : null
         })
         const captchaHash = await defaultHashUtil.hashPassword(captcha.text)
         const claims = { captcha: captchaHash }
@@ -99,7 +143,7 @@ export function CreateLocalIdpRoutes(localIdpApp: LocalIdpOperations, tokenServi
             ...parsedEnrollment,
             username
           }
-          const appRes = await localIdpApp.enrollMyself(enrollment)
+          const appRes = await ingressApp.enrollMyself(enrollment)
           if (appRes.success) {
             return res.json(appRes.success)
           }
diff --git a/service/src/ingress/ingress.protocol.bindings.ts b/service/src/ingress/ingress.protocol.bindings.ts
new file mode 100644
index 000000000..bc38ffd90
--- /dev/null
+++ b/service/src/ingress/ingress.protocol.bindings.ts
@@ -0,0 +1,12 @@
+import express from 'express'
+import { IdentityProvider } from './ingress.entities'
+
+/**
+ * `IngressProtocolWebBinding` is the binding of an authentication protocol's HTTP requests to an identity provider.
+ * The protocol uses the identity provider settings to determine the identity provider's endpoints and orchestrate the
+ * flow of HTTP messages between the Mage client, Mage server, and the identity provider's endpoints.
+ */
+export interface IngressProtocolWebBinding {
+  readonly idp: IdentityProvider
+  handleRequest: express.RequestHandler
+}
\ No newline at end of file

From ce94d64df09e527622eee361edf24858d09bb78d Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 22 Oct 2024 22:44:52 -0600
Subject: [PATCH 119/183] refactor(service): users/auth: cleanup comments

---
 service/src/entities/events/entities.events.ts | 3 ---
 service/src/entities/users/entities.users.ts   | 7 +++----
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/service/src/entities/events/entities.events.ts b/service/src/entities/events/entities.events.ts
index 967b1e619..0b57aa0a3 100644
--- a/service/src/entities/events/entities.events.ts
+++ b/service/src/entities/events/entities.events.ts
@@ -184,8 +184,6 @@ export interface MageEventRepository {
   findActiveEvents(): Promise<MageEvent[]>
   /**
    * Add a reference to the given feed ID on the given event.
-   * @param event an Event ID
-   * @param feed a Feed ID
    */
   addFeedsToEvent(event: MageEventId, ...feeds: FeedId[]): Promise<MageEvent | null>
   findTeamsInEvent(event: MageEventId): Promise<Team[] | null>
@@ -193,7 +191,6 @@ export interface MageEventRepository {
   /**
    * Remove the given feeds from any events that reference the feed.  Return the
    * count of events the operation modified.
-   * @param feed the ID of the feed to remove from events
    */
   removeFeedsFromEvents(...feed: FeedId[]): Promise<number>
 }
diff --git a/service/src/entities/users/entities.users.ts b/service/src/entities/users/entities.users.ts
index c0651b32e..824abbe9e 100644
--- a/service/src/entities/users/entities.users.ts
+++ b/service/src/entities/users/entities.users.ts
@@ -13,7 +13,7 @@ export interface User {
    */
   active: boolean
   /**
-   * The enabled flag indicates whether a user can access Mage and preform any operations.  An administrator can
+   * The enabled flag indicates whether a user can access Mage and perform any operations.  An administrator can
    * disable a user account at any time to block the user's access.
    */
   enabled: boolean
@@ -23,13 +23,12 @@ export interface User {
   email?: string
   phones: Phone[]
   /**
-   * A user's avatar is the profile picture that represents the user in list
-   * views and such.
+   * A user's avatar is the profile picture that represents the user in list views and such.
    * TODO: make this nullable rather than an empty object. that is a symptom of the mongoose schema. make sure a null value does not break clients
    */
   avatar: Avatar
   /**
-   * A user's icon is to indicate the user's location on a map display.
+   * The purpose of the user icon is to represent the user's location on a map display.
    */
   icon: UserIcon
   recentEventIds: MageEventId[]

From 8251bc8fa632c10ac432f74ff7b25e1a315a8f76 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 22 Oct 2024 22:50:51 -0600
Subject: [PATCH 120/183] refactor(service): users/auth: strongly type the
 permissions list in the role entity

---
 .../src/entities/authorization/entities.authorization.ts   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/service/src/entities/authorization/entities.authorization.ts b/service/src/entities/authorization/entities.authorization.ts
index 8c9502ea5..974b717a2 100644
--- a/service/src/entities/authorization/entities.authorization.ts
+++ b/service/src/entities/authorization/entities.authorization.ts
@@ -1,7 +1,10 @@
+import { AnyPermission } from './entities.permissions'
+
+export type RoleId = string
 
 export interface Role {
-  id: string
+  id: RoleId
   name: string
   description?: string
-  permissions: string[]
+  permissions: AnyPermission[]
 }

From d742293dadd4aa4525553457aeb4ab91c360d583 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 22 Oct 2024 22:54:22 -0600
Subject: [PATCH 121/183] refactor(service): users/auth: better semantic names
 for jwt token assertions

---
 service/src/ingress/verification.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/service/src/ingress/verification.ts b/service/src/ingress/verification.ts
index 3010b269f..6ea75b787 100644
--- a/service/src/ingress/verification.ts
+++ b/service/src/ingress/verification.ts
@@ -18,8 +18,8 @@ export enum VerificationErrorReason {
 }
 
 export enum TokenAssertion {
-  Authorized = 'urn:mage:auth:authorized',
-  Captcha = 'urn:mage:signup:captcha'
+  Authenticated = 'urn:mage:ingress:authenticated',
+  IsHuman = 'urn:mage:ingress:is_human'
 }
 
 export class TokenGenerateError extends Error {

From 7d6f46fa689a28da053bf315577534549d81227c Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 22 Oct 2024 22:54:51 -0600
Subject: [PATCH 122/183] refactor(service): users/auth: user admission test
 stubs

---
 service/test/app/ingress/app.ingress.test.ts | 49 ++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 service/test/app/ingress/app.ingress.test.ts

diff --git a/service/test/app/ingress/app.ingress.test.ts b/service/test/app/ingress/app.ingress.test.ts
new file mode 100644
index 000000000..bee7572fc
--- /dev/null
+++ b/service/test/app/ingress/app.ingress.test.ts
@@ -0,0 +1,49 @@
+import { expect } from 'chai'
+
+describe('ingress use cases', function() {
+
+  describe('admitting users', function() {
+
+    describe('without an existing account', function() {
+
+      it('creates a new account bound to the admitting idp', async function() {
+        expect.fail('todo')
+      })
+
+      it('applies enrollment policies to the new account', async function() {
+        expect.fail('todo')
+      })
+
+      it('generates an admission token', async function() {
+        expect.fail('todo')
+      })
+
+      it('fails if the enrollment policy requires account approval', async function() {
+        expect.fail('todo')
+      })
+
+      it('fails if the idp is disabled', async function() {
+        expect.fail('todo')
+      })
+    })
+
+    describe('with an existing account', function() {
+
+      it('generates an admission token', async function() {
+        expect.fail('todo')
+      })
+
+      it('fails without a matching idp binding', async function() {
+        expect.fail('todo')
+      })
+
+      it('fails if the idp is disabled', async function() {
+        expect.fail('todo')
+      })
+
+      it('fails if the account is disabled', async function() {
+        expect.fail('todo')
+      })
+    })
+  })
+})
\ No newline at end of file

From 87b7efb2d75f34bd744bcb8d4b97fc62ee243df0 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 24 Oct 2024 18:22:05 -0600
Subject: [PATCH 123/183] refactor(service): users/auth: cleanup local
 enrollment web flow types and separate routers for local enrollment and idp
 admission

---
 .../ingress.adapters.controllers.web.ts       | 108 +++++++++++++-----
 1 file changed, 78 insertions(+), 30 deletions(-)

diff --git a/service/src/ingress/ingress.adapters.controllers.web.ts b/service/src/ingress/ingress.adapters.controllers.web.ts
index d817c864c..6869c53af 100644
--- a/service/src/ingress/ingress.adapters.controllers.web.ts
+++ b/service/src/ingress/ingress.adapters.controllers.web.ts
@@ -3,7 +3,7 @@ import svgCaptcha from 'svg-captcha'
 import { Authenticator } from 'passport'
 import { Strategy as BearerStrategy } from 'passport-http-bearer'
 import { defaultHashUtil } from '../utilities/password-hashing'
-import { JWTService, Payload, TokenVerificationError, VerificationErrorReason } from './verification'
+import { JWTService, Payload, TokenVerificationError, VerificationErrorReason, TokenAssertion } from './verification'
 import { invalidInput, InvalidInputError, MageError } from '../app.api/app.api.errors'
 import { IdentityProviderRepository } from './ingress.entities'
 import { AdmitFromIdentityProviderOperation, EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
@@ -11,41 +11,61 @@ import { IngressProtocolWebBinding } from './ingress.protocol.bindings'
 
 declare module 'express-serve-static-core' {
   interface Request {
-    identityProviderService?: IngressProtocolWebBinding
+    ingress?: IngressRequestContext
   }
 }
 
+type IngressRequestContext = { identityProviderService: IngressProtocolWebBinding } & (
+  | { state: 'init' }
+  | { state: 'localEnrollment', localEnrollment: LocalEnrollment }
+)
+
+type LocalEnrollment =
+  | {
+    state: 'humanTokenVerified'
+    captchaTokenPayload: Payload
+  }
+  | {
+    state: 'humanVerified'
+    subject: string
+  }
+
 export type IngressOperations = {
   enrollMyself: EnrollMyselfOperation
   admitFromIdentityProvider: AdmitFromIdentityProviderOperation
 }
 
+export type IngressRoutes = {
+  localEnrollment: express.Router
+  idpAdmission: express.Router
+}
+
 function bindingFor(idpName: string): IngressProtocolWebBinding {
   throw new Error('unimplemented')
 }
 
-export function CreateIngressRoutes(ingressApp: IngressOperations, idpRepo: IdentityProviderRepository, tokenService: JWTService, passport: Authenticator): express.Router {
+export function CreateIngressRoutes(ingressApp: IngressOperations, idpRepo: IdentityProviderRepository, tokenService: JWTService, passport: Authenticator): IngressRoutes {
 
   const captchaBearer = new BearerStrategy((token, done) => {
     const expectation = {
       subject: null,
       expiration: null,
-      assertion: TokenAssertion.Captcha
+      assertion: TokenAssertion.IsHuman
     }
     tokenService.verifyToken(token, expectation)
       .then(payload => done(null, payload))
       .catch(err => done(err))
   })
 
-  const routes = express.Router()
-
-  // TODO: signup
-  // TODO: signin
+  // TODO: separate routers for /auth/idp/* and /api/users/signups/* for backward compatibility
 
   const routeToIdp = express.Router().all('/',
     ((req, res, next) => {
-      const idpService = req.identityProviderService!
-      idpService.handleRequest(req, res, next)
+      const idpService = req.ingress?.identityProviderService
+      if (idpService) {
+        return idpService.handleRequest(req, res, next)
+      }
+      next(new Error(`no identity provider for ingress request: ${req.method} ${req.originalUrl}`))
     }) as express.RequestHandler,
     (async (err, req, res, next) => {
       if (err) {
@@ -56,34 +76,53 @@ export function CreateIngressRoutes(ingressApp: IngressOperations, idpRepo: Iden
         console.error('unexpected authentication user type:', req.user?.from)
         return res.status(500).send('unexpected authentication result')
       }
-      const identityProviderName = req.identityProviderService!.idp.name
+      const identityProviderName = req.ingress!.identityProviderService!.idp.name
       const identityProviderUser = req.user.account
-      const ingressResult = await ingressApp.admitFromIdentityProvider({ identityProviderName, identityProviderUser })
-      if (ingressResult.error) {
-        next(ingressResult.error)
+      const admission = await ingressApp.admitFromIdentityProvider({ identityProviderName, identityProviderUser })
+      if (admission.error) {
+        return next(admission.error)
+      }
+      const { admissionToken, mageAccount } = admission.success
+      /*
+      TODO: copied from redirecting protocols - cleanup and adapt here
+      local/ldap use direct json response
+      saml uses RelayState body property
+      oauth/oidc use state query parameter
+      can all use direct json response and handle redirect windows client side?
+      */
+      if (req.query.state === 'mobile') {
+        let uri;
+        if (!mageAccount.active || !mageAccount.enabled) {
+          uri = `mage://app/invalid_account?active=${mageAccount.active}&enabled=${mageAccount.enabled}`;
+        } else {
+          uri = `mage://app/authentication?token=${req.token}`
+        }
+        res.redirect(uri);
+      } else {
+        res.render('authentication', { host: req.getRoot(), success: true, login: { token: req.token, user: req.user } });
       }
-      // if user active and enabled, send authenticated JWT and proceed to verification
-      // else
-      const account = ingressResult.success!
-
     }) as express.ErrorRequestHandler
   )
 
-  routes.use('/:identityProviderName',
+  // TODO: mount to /auth
+  const idpAdmission = express.Router()
+  idpAdmission.use('/:identityProviderName',
     (req, res, next) => {
       const idpName = req.params.identityProviderName
       const idpService = bindingFor(idpName)
       if (idpService) {
-        req.identityProviderService = idpService
+        req.ingress = { state: 'init', identityProviderService: idpService }
         return next()
       }
       res.status(404).send(`${idpName} not found`)
     },
+    // use a sub-router so express implicitly strips the base url /auth/:identityProviderName before routing idp handler
     routeToIdp
   )
 
   // TODO: mount to /api/users/signups
-  routes.route('/signups')
+  const localEnrollment = express.Router()
+  localEnrollment.route('/signups')
     .post(async (req, res, next) => {
       try {
         const username = typeof req.body.username === 'string' ? req.body.username.trim() : null
@@ -99,7 +138,7 @@ export function CreateIngressRoutes(ingressApp: IngressOperations, idpRepo: Iden
         })
         const captchaHash = await defaultHashUtil.hashPassword(captcha.text)
         const claims = { captcha: captchaHash }
-        const verificationToken = await tokenService.generateToken(username, TokenAssertion.Captcha, 60 * 3, claims)
+        const verificationToken = await tokenService.generateToken(username, TokenAssertion.IsHuman, 60 * 3, claims)
         res.json({
           token: verificationToken,
           captcha: `data:image/svg+xml;base64,${Buffer.from(captcha.data).toString('base64')}`
@@ -110,7 +149,8 @@ export function CreateIngressRoutes(ingressApp: IngressOperations, idpRepo: Iden
       }
     })
 
-  routes.route('/signups/verifications')
+  // TODO: mount to /api/users/signups/verifications
+  localEnrollment.route('/signups/verifications')
     .post(
       async (req, res, next) => {
         passport.authenticate(captchaBearer, (err: TokenVerificationError, captchaTokenPayload: Payload) => {
@@ -123,18 +163,26 @@ export function CreateIngressRoutes(ingressApp: IngressOperations, idpRepo: Iden
           if (!captchaTokenPayload) {
             return res.status(400).send('Missing captcha token')
           }
-          req.user = captchaTokenPayload
+          req.ingress = {
+            ...req.ingress!,
+            state: 'localEnrollment',
+            localEnrollment: { state: 'humanTokenVerified', captchaTokenPayload } }
           next()
         })(req, res, next)
       },
       async (req, res, next) => {
         try {
-          const isHuman = await defaultHashUtil.validPassword(req.body.captchaText, req.user.captcha)
+          if (req.ingress?.state !== 'localEnrollment' || req.ingress.localEnrollment.state !== 'humanTokenVerified') {
+            return res.status(500).send('invalid ingress state')
+          }
+          const tokenPayload = req.ingress.localEnrollment.captchaTokenPayload
+          const hashedCaptchaText = tokenPayload.captcha as string
+          const userCaptchaText = req.body.captchaText
+          const isHuman = await defaultHashUtil.validPassword(userCaptchaText, hashedCaptchaText)
           if (!isHuman) {
             return res.status(403).send('Invalid captcha. Please try again.')
           }
-          const payload = req.user as Payload
-          const username = payload.subject!
+          const username = tokenPayload.subject!
           const parsedEnrollment = validateEnrollment(req.body)
           if (parsedEnrollment instanceof MageError) {
             return next(parsedEnrollment)
@@ -155,15 +203,15 @@ export function CreateIngressRoutes(ingressApp: IngressOperations, idpRepo: Iden
       }
     )
 
-  return routes
+  return { localEnrollment, idpAdmission }
 }
 
 function validateEnrollment(input: any): Omit<EnrollMyselfRequest, 'username'> | InvalidInputError {
   const { displayName, email, password, phone } = input
-  if (!displayName) {
+  if (typeof displayName !== 'string') {
     return invalidInput('displayName is required')
   }
-  if (!password) {
+  if (typeof password !== 'string') {
     return invalidInput('password is required')
   }
   const enrollment: Omit<EnrollMyselfRequest, 'username'> = { displayName, password }

From 4a6ff2639570ad3475c76d7fefc44bbf207339d8 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 28 Oct 2024 11:36:44 -0600
Subject: [PATCH 124/183] refactor(service): users/auth: cleanup local auth
 protocol handler names and initialize only a single express router instead of
 an array of handlers

---
 service/src/ingress/ingress.protocol.local.ts | 23 +++++++++----------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.local.ts b/service/src/ingress/ingress.protocol.local.ts
index fb9163b18..551fcc617 100644
--- a/service/src/ingress/ingress.protocol.local.ts
+++ b/service/src/ingress/ingress.protocol.local.ts
@@ -14,7 +14,7 @@ function userForLocalIdpAccount(account: LocalIdpAccount): IdentityProviderUser
   }
 }
 
-function createProtocolMiddleware(localIdpAuthenticate: LocalIdpAuthenticateOperation): passport.Strategy {
+function createAuthenticationMiddleware(localIdpAuthenticate: LocalIdpAuthenticateOperation): passport.Strategy {
   const verify: LocalStrategyVerifyFunction = async function LocalIngressProtocolVerify(username, password, done) {
     const authResult = await localIdpAuthenticate({ username, password })
     if (authResult.success) {
@@ -27,7 +27,7 @@ function createProtocolMiddleware(localIdpAuthenticate: LocalIdpAuthenticateOper
   return new LocalStrategy(verify)
 }
 
-const validateRequest: express.RequestHandler = function LocalProtocolIngressHandler(req, res, next) {
+const validateSigninRequest: express.RequestHandler = function LocalProtocolIngressHandler(req, res, next) {
   if (req.method !== 'POST' || !req.body) {
     return res.status(400).send(`invalid request method ${req.method}`)
   }
@@ -39,14 +39,13 @@ const validateRequest: express.RequestHandler = function LocalProtocolIngressHan
   next()
 }
 
-export function createIngressStack(passport: passport.Authenticator, localIdpAuthenticate: LocalIdpAuthenticateOperation): express.RequestHandler[] {
-  const authStrategy = createProtocolMiddleware(localIdpAuthenticate)
-  const authHandler: express.RequestHandler = async (req, res, next) => {
-    passport.authenticate(authStrategy)(req, res, next)
-  }
-  return [
-    express.urlencoded(),
-    validateRequest,
-    authHandler,
-  ]
+export function createWebBinding(passport: passport.Authenticator, localIdpAuthenticate: LocalIdpAuthenticateOperation): express.RequestHandler {
+  const authStrategy = createAuthenticationMiddleware(localIdpAuthenticate)
+  const handleRequest = express.Router()
+    .post('/signin',
+      express.urlencoded(),
+      validateSigninRequest,
+      passport.authenticate(authStrategy)
+    )
+  return handleRequest
 }

From f5830a8ef603e5642a27aa00215aa54efd021ff4 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 28 Oct 2024 11:37:26 -0600
Subject: [PATCH 125/183] refactor(service): users/auth: migrate old oauth
 strategy to new protocol architecture

---
 service/src/ingress/ingress.protocol.oauth.ts | 233 +++++-------------
 1 file changed, 63 insertions(+), 170 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.oauth.ts b/service/src/ingress/ingress.protocol.oauth.ts
index ccadcd63d..c8c64d8d2 100644
--- a/service/src/ingress/ingress.protocol.oauth.ts
+++ b/service/src/ingress/ingress.protocol.oauth.ts
@@ -1,14 +1,9 @@
-'use strict';
-
-import { InternalOAuthError, Strategy as OAuth2Strategy, StrategyOptions as OAuth2Options, VerifyFunction } from 'passport-oauth2'
-import { TokenAssertion, JWTService } from './verification'
+import express from 'express'
+import { InternalOAuthError, Strategy as OAuth2Strategy, StrategyOptions as OAuth2Options, VerifyCallback, VerifyFunction } from 'passport-oauth2'
 import base64 from 'base-64'
-import { IdentityProvider } from './entities.authentication'
+import { IdentityProvider, IdentityProviderUser } from './ingress.entities'
 import { Authenticator } from 'passport'
-const api = require('../api')
-const log = require('../logger')
-const User = require('../models/user')
-const Role = require('../models/role')
+import { WebIngressUserFromIdentityProvider } from '../@types/express'
 
 export type OAuth2ProtocolSettings =
   Pick<OAuth2Options,
@@ -24,12 +19,25 @@ export type OAuth2ProtocolSettings =
     headers?: { basic?: boolean | null | undefined },
     profile: OAuth2ProfileKeys
   }
+
 export type OAuth2ProfileKeys = {
   id: string
   email: string
   displayName: string
 }
 
+function copyProtocolSettings(from: OAuth2ProtocolSettings): OAuth2ProtocolSettings {
+  const copy = { ...from }
+  copy.profile = { ...from.profile }
+  if (Array.isArray(from.scope)) {
+    copy.scope = [ ...from.scope ]
+  }
+  if (from.headers) {
+    copy.headers = { ...from.headers }
+  }
+  return copy
+}
+
 class OAuth2ProfileStrategy extends OAuth2Strategy {
 
   constructor(options: OAuth2Options, readonly profileURL: string, verify: VerifyFunction) {
@@ -59,19 +67,38 @@ class OAuth2ProfileStrategy extends OAuth2Strategy {
   }
 }
 
-function configure(strategy: IdentityProvider, passport: Authenticator, ) {
-  log.info(`configuring ${strategy.title} oauth2 authentication`);
-  const settings = strategy.protocolSettings as OAuth2ProtocolSettings
+function applyDefaultProtocolSettings(idp: IdentityProvider): OAuth2ProtocolSettings {
+  const settings = copyProtocolSettings(idp.protocolSettings as OAuth2ProtocolSettings)
+  const profile = settings.profile
+  if (!profile.displayName) {
+    profile.displayName = 'displayName'
+  }
+  if (!profile.email) {
+    profile.email = 'email'
+  }
+  if (!profile.id) {
+    profile.id = 'id';
+  }
+  return settings
+}
+
+/**
+ * The `baseUrl` parameter is the URL at which Mage will mount the returned `express.Router`, including any
+ * distinguishing component of the given `IdentityProvider`, without a trailing slash, e.g. `/auth/example-idp`.
+ */
+export function createWebBinding(idp: IdentityProvider, passport: Authenticator, baseUrl: string): express.Router {
+  const settings = applyDefaultProtocolSettings(idp)
+  const profileURL = settings.profileURL
   const customHeaders = settings.headers?.basic ? {
     authorization: `Basic ${base64.encode(`${settings.clientID}:${settings.clientSecret}`)}`
   } : undefined
   const strategyOptions: OAuth2Options = {
     clientID: settings.clientID,
     clientSecret: settings.clientSecret,
-    callbackURL: `/auth/${strategy.name}/callback`,
+    callbackURL: `${baseUrl}/callback`,
     authorizationURL: settings.authorizationURL,
     tokenURL: settings.tokenURL,
-    customHeaders: customHeaders,
+    customHeaders,
     scope: settings.scope,
     pkce: settings.pkce,
     /**
@@ -80,166 +107,32 @@ function configure(strategy: IdentityProvider, passport: Authenticator, ) {
      */
     store: true as any
   }
-  const verify: VerifyFunction = (accessToken, refreshToken, profileResponse, done) => {
+  const verify: VerifyFunction = (accessToken: string, refreshToken: string, profileResponse: any, done: VerifyCallback) => {
     const profile = profileResponse.json
     const profileKeys = settings.profile
     if (!profile[profileKeys.id]) {
       log.warn("JSON: " + JSON.stringify(profile) + " RAW: " + profileResponse.raw);
-      return done(`OAuth2 user profile does not contain id property named ${profileKeys.id}`);
-    }
-
-    const profileId = profile[settings.profile.id];
-
-    // TODO: users-next
-    // TODO: should be by strategy name, not strategy type
-    User.getUserByAuthenticationStrategy(strategy.type, profileId, function (err, user) {
-      if (err) return done(err);
-
-      if (!user) {
-        // Create an account for the user
-        Role.getRole('USER_ROLE', function (err, role) {
-          if (err) {
-            return done(err)
-          }
-          const profileEmail = profile[profileKeys.email]
-          if (profile[profileKeys.email]) {
-            if (Array.isArray(profile[profileKeys.email])) {
-              email = profile[profileKeys.email].find(email => {
-                email.verified === true
-              });
-            } else {
-              email = profile[settings.profile.email];
-            }
-          } else {
-            log.warn(`OAuth2 user profile does not contain email property named ${profileKeys.email}`);
-            log.debug(JSON.stringify(profile));
-          }
-
-          const user = {
-            username: profileId,
-            displayName: profile[profileKeys.displayName] || profileId,
-            email: email,
-            active: false,
-            roleId: role._id,
-            authentication: {
-              type: strategy.type,
-              id: profileId,
-              authenticationConfiguration: {
-                name: strategy.name
-              }
-            }
-          };
-          // TODO: users-next
-          new api.User().create(user).then(newUser => {
-            if (!newUser.authentication.authenticationConfiguration.enabled) {
-              log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
-              return done(null, false, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-            }
-            return done(null, newUser);
-          }).catch(err => done(err));
-        });
-      } else if (!user.active) {
-        return done(null, user, { message: "User is not approved, please contact your MAGE administrator to approve your account." });
-      } else if (!user.authentication.authenticationConfiguration.enabled) {
-        log.warn(user.authentication.authenticationConfiguration.title + " authentication is not enabled");
-        return done(null, user, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-      } else {
-        return done(null, user);
-      }
-    });
-  }
-  const oauth2Strategy = new OAuth2ProfileStrategy(strategyOptions, verify)
-
-}
-
-function setDefaults(strategy) {
-  if (!strategy.settings.profile) {
-    strategy.settings.profile = {};
-  }
-  if (!strategy.settings.profile.displayName) {
-    strategy.settings.profile.displayName = 'displayName';
-  }
-  if (!strategy.settings.profile.email) {
-    strategy.settings.profile.email = 'email';
-  }
-  if (!strategy.settings.profile.id) {
-    strategy.settings.profile.id = 'id';
-  }
-}
-
-function initialize(strategy) {
-  setDefaults(strategy);
-
-  // TODO lets test with newer geoaxis server to see if this is still needed
-  // If it is, this should be a admin client side option, would also need to modify the
-  // renderer to provide a more generic message
-  strategy.redirect = false;
-  configure(strategy);
-
-  function authenticate(req, res, next) {
-    passport.authenticate(strategy.name, function (err, user, info = {}) {
-      if (err) return next(err);
-
-      req.user = user;
-
-      // For inactive or disabled accounts don't generate an authorization token
-      if (!user.active || !user.enabled) {
-        log.warn('Failed user login attempt: User ' + user.username + ' account is inactive or disabled.');
-        return next();
-      }
-
-      if (!user.authentication.authenticationConfigurationId) {
-        log.warn('Failed user login attempt: ' + user.authentication.type + ' is not configured');
-        return next();
-      }
-
-      if (!user.authentication.authenticationConfiguration.enabled) {
-        log.warn('Failed user login attempt: Authentication ' + user.authentication.authenticationConfiguration.title + ' is disabled.');
-        return next();
-      }
-
-      tokenService.generateToken(user._id.toString(), TokenAssertion.Authorized, 60 * 5)
-      .then(token => {
-        req.token = token;
-        req.user = user;
-        req.info = info;
-        next();
-      }).catch(err => next(err));
-    })(req, res, next);
-  }
-
-  app.get(`/auth/${strategy.name}/signin`,
-    function (req, res, next) {
-      passport.authenticate(strategy.name, {
-        scope: strategy.settings.scope,
-        state: req.query.state
-      })(req, res, next);
+      return done(`OAuth2 user profile does not contain id property named ${profileKeys.id}`)
     }
-  );
-
-  app.get(`/auth/${strategy.name}/callback`,
-    authenticate,
-    function (req, res) {
-      if (req.query.state === 'mobile') {
-        let uri;
-        if (!req.user.active || !req.user.enabled) {
-          uri = `mage://app/invalid_account?active=${req.user.active}&enabled=${req.user.enabled}`;
-        } else {
-          uri = `mage://app/authentication?token=${req.token}`
-        }
-
-        if (strategy.redirect) {
-          res.redirect(uri);
-        } else {
-          res.render('oauth', { uri: uri });
-        }
-      } else {
-        res.render('authentication', { host: req.getRoot(), success: true, login: { token: req.token, user: req.user } });
-      }
+    const username = profile[profileKeys.id]
+    const displayName = profile[profileKeys.displayName] || username
+    const email = profile[profileKeys.email]
+    const idpUser: IdentityProviderUser = { username, displayName, email, phones: [] }
+    const ingressUser: WebIngressUserFromIdentityProvider = {
+      from: 'identityProvider',
+      account: idpUser
     }
-  );
-};
-
-module.exports = {
-  initialize
+    return done(null, ingressUser)
+  }
+  const oauth2Strategy = new OAuth2ProfileStrategy(strategyOptions, profileURL, verify)
+  return express.Router()
+    .get('/signin',
+      /*
+      TODO:
+      this used to pass state in the options argument like { state: req.query.state }
+      to propagate the mobile clients passing state=mobile. test whether this is actually necessary
+      */
+      passport.authenticate(oauth2Strategy, { scope: settings.scope })
+    )
+    .get('/callback', passport.authenticate(oauth2Strategy))
 }
\ No newline at end of file

From 29db301d34039217f2194c0a00e9a59be30a51b1 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 28 Oct 2024 21:57:21 -0600
Subject: [PATCH 126/183] refactor(service): users/auth: remove unused types

---
 service/src/ingress/ingress.app.impl.ts | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/service/src/ingress/ingress.app.impl.ts b/service/src/ingress/ingress.app.impl.ts
index 32ebcf0d1..af325c692 100644
--- a/service/src/ingress/ingress.app.impl.ts
+++ b/service/src/ingress/ingress.app.impl.ts
@@ -4,7 +4,7 @@ import { MageEventId } from '../entities/events/entities.events'
 import { Team, TeamId } from '../entities/teams/entities.teams'
 import { User, UserId, UserRepository, UserRepositoryError } from '../entities/users/entities.users'
 import { AdmitFromIdentityProviderOperation, AdmitFromIdentityProviderRequest, authenticationFailedError, EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
-import { createEnrollmentCandidateUser, IdentityProvider, IdentityProviderRepository, IdentityProviderUser, UserIngressBinding, UserIngressBindingRepository, UserIngressBindings } from './ingress.entities'
+import { createEnrollmentCandidateUser, IdentityProvider, IdentityProviderRepository, IdentityProviderUser, UserIngressBindingRepository, UserIngressBindings } from './ingress.entities'
 import { LocalIdpCreateAccountOperation } from './local-idp.app.api'
 import { JWTService, TokenAssertion } from './verification'
 
@@ -40,13 +40,6 @@ export interface FindEventTeam {
   (mageEventId: MageEventId): Promise<Team | null>
 }
 
-type TeamAssignmentResult = { teamId: TeamId, assigned: boolean }
-type EventAssignmentResult = { eventId: MageEventId, teamId: TeamId | null, assigned: boolean }
-type EnrollmentTeamAssignmentResult = {
-  teamAssignments: TeamAssignmentResult[]
-  eventAssignments: EventAssignmentResult[]
-}
-
 async function enrollNewUser(idpAccount: IdentityProviderUser, idp: IdentityProvider, userRepo: UserRepository, ingressBindingRepo: UserIngressBindingRepository, findEventTeam: FindEventTeam, assignTeamMember: AssignTeamMember): Promise<{ mageAccount: User, ingressBindings: UserIngressBindings }> {
   console.info(`enrolling new user account ${idpAccount.username} from identity provider ${idp.name}`)
   const candidate = createEnrollmentCandidateUser(idpAccount, idp)

From 6e53f18ca4ce57f5f6849c73b112a0467f21c777 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 31 Oct 2024 12:27:39 -0600
Subject: [PATCH 127/183] refactor(service): users/auth: port some init code in
 app.ts to new user components

---
 service/src/app.ts | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/service/src/app.ts b/service/src/app.ts
index 01de1b53c..42d988b29 100644
--- a/service/src/app.ts
+++ b/service/src/app.ts
@@ -38,7 +38,7 @@ import { MageEventRepositoryToken } from './plugins.api/plugins.api.events'
 import { FeedRepositoryToken, FeedServiceRepositoryToken, FeedServiceTypeRepositoryToken, FeedsAppServiceTokens } from './plugins.api/plugins.api.feeds'
 import { UserRepositoryToken } from './plugins.api/plugins.api.users'
 import { StaticIconRepositoryToken } from './plugins.api/plugins.api.icons'
-import { UserModel, MongooseUserRepository } from './adapters/users/adapters.users.db.mongoose'
+import { UserModel, MongooseUserRepository, UserModelName } from './adapters/users/adapters.users.db.mongoose'
 import { UserRepository, UserExpanded } from './entities/users/entities.users'
 import { EnvironmentService } from './entities/systemInfo/entities.systemInfo'
 import { WebRoutesHooks, GetAppRequestContext } from './plugins.api/plugins.api.web'
@@ -61,9 +61,6 @@ import { EnvironmentServiceImpl } from './adapters/systemInfo/adapters.systemInf
 import { SystemInfoAppLayer } from './app.api/systemInfo/app.api.systemInfo'
 import { CreateReadSystemInfo } from './app.impl/systemInfo/app.impl.systemInfo'
 import Settings from "./models/setting";
-// TODO: users-next
-import AuthenticationConfiguration from "./models/authenticationconfiguration";
-import AuthenticationConfigurationTransformer from "./transformers/authenticationconfiguration";
 import { SystemInfoRoutes } from './adapters/systemInfo/adapters.systemInfo.controllers.web'
 import { RoleBasedSystemInfoPermissionService } from './permissions/permissions.systemInfo'
 import { SettingsAppLayer, SettingsRoutes } from './adapters/settings/adapters.settings.controllers.web'
@@ -252,6 +249,7 @@ type AppLayer = {
     allocateObservationId: observationsApi.AllocateObservationId
     saveObservation: observationsApi.SaveObservation
     readObservation: observationsApi.ReadObservation
+    readObservations: observationsApi.ReadObservations
     storeAttachmentContent: observationsApi.StoreAttachmentContent
     readAttachmentContent: observationsApi.ReadAttachmentContent
   },
@@ -322,7 +320,7 @@ async function initDatabase(): Promise<DatabaseLayer> {
       staticIcon: StaticIconModel(conn)
     },
     users: {
-      user: require('./models/user').Model
+      user: UserModel(conn)
     },
     settings: {
       setting: require('./models/setting').Model
@@ -465,6 +463,7 @@ async function initObservationsAppLayer(repos: Repositories): Promise<AppLayer['
     allocateObservationId: observationsImpl.AllocateObservationId(obsPermissionsService),
     saveObservation: observationsImpl.SaveObservation(obsPermissionsService, repos.users.userRepo),
     readObservation: observationsImpl.ReadObservation(obsPermissionsService),
+    readObservations: observationsImpl.ReadObservations(obsPermissionsService),
     storeAttachmentContent: observationsImpl.StoreAttachmentContent(obsPermissionsService, repos.observations.attachmentStore),
     readAttachmentContent: observationsImpl.ReadAttachmentContent(obsPermissionsService, repos.observations.attachmentStore)
   }
@@ -524,6 +523,7 @@ function initSystemInfoAppLayer(repos: Repositories): SystemInfoAppLayer {
       repos.enviromentInfo,
       versionInfo,
       Settings,
+      // TODO: users-next: new auth elements
       AuthenticationConfiguration,
       AuthenticationConfigurationTransformer,
       permissionsService
@@ -599,6 +599,7 @@ async function initWebLayer(repos: Repositories, app: AppLayer, webUIPlugins: st
     const context: observationsApi.ObservationRequestContext = {
       ...baseAppRequestContext(req),
       mageEvent: req[observationEventScopeKey]!.mageEvent,
+      // TODO: users-next
       userId: req.user.id,
       deviceId: req.provisionedDeviceId,
       observationRepository: req[observationEventScopeKey]!.observationRepository
@@ -644,7 +645,7 @@ async function initWebLayer(repos: Repositories, app: AppLayer, webUIPlugins: st
         middleware applies the entity form of a user on the request rather than
         the mongoose document instance
         */
-        return { ...req.user.toJSON(), id: req.user._id.toHexString() } as UserExpanded
+        return { ...req.user?.account as UserExpanded }
       },
       locale(): Locale | null {
         return Object.freeze({

From ed9a1b141b644479547d8230893b27b334a24561 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 31 Oct 2024 12:30:19 -0600
Subject: [PATCH 128/183] refactor(service): users/auth: change app response
 class to typedef to allow better type inference of mutually exclusive
 nullability of success and error

---
 service/src/app.api/app.api.global.ts | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/service/src/app.api/app.api.global.ts b/service/src/app.api/app.api.global.ts
index a0a3f1642..2ea5ec2f3 100644
--- a/service/src/app.api/app.api.global.ts
+++ b/service/src/app.api/app.api.global.ts
@@ -51,17 +51,17 @@ export interface Descriptor<T extends string> extends JsonObject {
  */
 export type AnyMageError<KnownErrors> = KnownErrors extends MageError<infer Code, infer Data> ? MageError<Code, Data> : never
 
-export class AppResponse<Success, KnownErrors> {
+export const AppResponse = Object.freeze({
 
-  static success<Success, KnownErrors>(result: Success, contentLanguage?: LanguageTag[] | null | undefined): AppResponse<Success, AnyMageError<KnownErrors>> {
-    return new AppResponse<Success, AnyMageError<KnownErrors>>(result, null, contentLanguage)
-  }
+  success<Success, KnownErrors>(result: Success, contentLanguage?: LanguageTag[] | null | undefined): AppResponse<Success, AnyMageError<KnownErrors>> {
+    return Object.freeze({ success: result, error: null,  contentLanguage })
+  },
 
-  static error<KnownErrors>(result: KnownErrors, contentLanguage?: LanguageTag[] | null | undefined): AppResponse<any, KnownErrors> {
-    return new AppResponse<any, KnownErrors>(null, result, contentLanguage)
-  }
+  error<KnownErrors>(result: KnownErrors, contentLanguage?: LanguageTag[] | null | undefined): AppResponse<any, KnownErrors> {
+    return Object.freeze({ success: null, error: result, contentLanguage })
+  },
 
-  static resultOf<Success, KnownErrors>(promise: Promise<Success | AnyMageError<KnownErrors>>): Promise<AppResponse<Success, AnyMageError<KnownErrors>>> {
+  resultOf<Success, KnownErrors>(promise: Promise<Success | AnyMageError<KnownErrors>>): Promise<AppResponse<Success, AnyMageError<KnownErrors>>> {
     return promise.then(
       successOrKnownError => {
         if (successOrKnownError instanceof MageError) {
@@ -70,9 +70,11 @@ export class AppResponse<Success, KnownErrors> {
         return AppResponse.success(successOrKnownError)
       })
   }
+})
 
-  private constructor(readonly success: Success | null, readonly error: KnownErrors | null, readonly contentLanguage?: LanguageTag[] | null | undefined) {}
-}
+export type AppResponse<Success, KnownErrors> =
+  & { contentLanguage?: LanguageTag[] | null | undefined }
+  & ({ success: Success, error: null } | { success: null, error: KnownErrors })
 
 /**
  * This type provides a shorthand to map the given operation type argument to

From 01c01c88d2c8721572710eaa226719fb73a45b4f Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 31 Oct 2024 12:38:15 -0600
Subject: [PATCH 129/183] refactor(service): users/auth: change default generic
 parameter type in web app request factory from object to record

---
 service/src/adapters/adapters.controllers.web.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/adapters/adapters.controllers.web.ts b/service/src/adapters/adapters.controllers.web.ts
index 0ece6201b..b9af0ea54 100644
--- a/service/src/adapters/adapters.controllers.web.ts
+++ b/service/src/adapters/adapters.controllers.web.ts
@@ -3,7 +3,7 @@ import { ErrEntityNotFound, ErrInfrastructure, ErrInvalidInput, ErrPermissionDen
 import { AppRequest } from '../app.api/app.api.global'
 
 export interface WebAppRequestFactory<Req extends AppRequest = AppRequest> {
-  <RequestParams extends object = object>(webReq: express.Request, params?: RequestParams): Req & RequestParams
+  <RequestParams extends Record<string, any> = Record<string, any>>(webReq: express.Request, params?: RequestParams): Req & RequestParams
 }
 
 /**

From 8425fccc2a5d2ad6ef6f6c403b8afa80378a02e1 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 31 Oct 2024 13:38:32 -0600
Subject: [PATCH 130/183] style(service): users/auth: remove obsolete comment

---
 service/src/ingress/sessions.adapters.db.mongoose.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/service/src/ingress/sessions.adapters.db.mongoose.ts b/service/src/ingress/sessions.adapters.db.mongoose.ts
index 56eecedec..debf13278 100644
--- a/service/src/ingress/sessions.adapters.db.mongoose.ts
+++ b/service/src/ingress/sessions.adapters.db.mongoose.ts
@@ -25,7 +25,6 @@ const SessionSchema = new Schema<SessionDocument, SessionModel>(
   { versionKey: false }
 )
 
-// TODO: index token
 SessionSchema.index({ token: 1, unique: 1 })
 SessionSchema.index({ expirationDate: 1 }, { expireAfterSeconds: 0 })
 

From 9284652e2102b42f8dba201e4de929ca081b4854 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 31 Oct 2024 13:39:27 -0600
Subject: [PATCH 131/183] refactor(service): users/auth: limit update
 attributes of identity providers

---
 service/src/ingress/ingress.entities.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/service/src/ingress/ingress.entities.ts b/service/src/ingress/ingress.entities.ts
index 21cba76db..9c6f27f4d 100644
--- a/service/src/ingress/ingress.entities.ts
+++ b/service/src/ingress/ingress.entities.ts
@@ -96,6 +96,8 @@ export interface UserIngressBinding {
   idpAccountAttrs?: Record<string, any>
 }
 
+export type IdentityProviderMutableAttrs = Omit<IdentityProvider, 'id' | 'name' | 'protocol'>
+
 export interface IdentityProviderRepository {
   findIdpById(id: IdentityProviderId): Promise<IdentityProvider | null>
   findIdpByName(name: string): Promise<IdentityProvider | null>
@@ -103,7 +105,7 @@ export interface IdentityProviderRepository {
    * Update the IDP according to patch semantics.  Remove keys in the given update with `undefined` values from the
    * saved record.  Keys not present in the given update will have no affect on the saved record.
    */
-  updateIdp(update: Partial<IdentityProvider> & Pick<IdentityProvider, 'id'>): Promise<IdentityProvider | null>
+  updateIdp(update: Partial<IdentityProviderMutableAttrs> & Pick<IdentityProvider, 'id'>): Promise<IdentityProvider | null>
   deleteIdp(id: IdentityProviderId): Promise<IdentityProvider | null>
 }
 

From 2142181b8c0eb3fff3e338f814c1c031b969644d Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 31 Oct 2024 15:54:32 -0600
Subject: [PATCH 132/183] refactor(service): users/auth: extract common
 enrollment function to service layer

---
 service/src/ingress/ingress.app.api.ts       | 13 ++++
 service/src/ingress/ingress.app.impl.ts      | 78 ++++----------------
 service/src/ingress/ingress.services.api.ts  |  6 ++
 service/src/ingress/ingress.services.impl.ts | 64 ++++++++++++++++
 4 files changed, 96 insertions(+), 65 deletions(-)
 create mode 100644 service/src/ingress/ingress.services.api.ts
 create mode 100644 service/src/ingress/ingress.services.impl.ts

diff --git a/service/src/ingress/ingress.app.api.ts b/service/src/ingress/ingress.app.api.ts
index a663bfc36..c926c9e42 100644
--- a/service/src/ingress/ingress.app.api.ts
+++ b/service/src/ingress/ingress.app.api.ts
@@ -4,6 +4,7 @@ import { Avatar, User, UserExpanded, UserIcon } from '../entities/users/entities
 import { IdentityProviderUser } from './ingress.entities'
 
 
+
 export interface EnrollMyselfRequest {
   username: string
   password: string
@@ -53,6 +54,18 @@ export interface AdmitFromIdentityProviderOperation {
   (req: AdmitFromIdentityProviderRequest): Promise<AppResponse<AdmitFromIdentityProviderResult, EntityNotFoundError | AuthenticationFailedError | InfrastructureError>>
 }
 
+export interface UpdateIdentityProviderRequest {
+
+}
+
+export interface UpdateIdentityProviderResult {
+
+}
+
+export interface UpdateIdentityProviderOperation {
+  (req: UpdateIdentityProviderRequest): Promise<AppResponse<UpdateIdentityProviderResult, PermissionDeniedError | EntityNotFoundError>>
+}
+
 export const ErrAuthenticationFailed = Symbol.for('MageError.Ingress.AuthenticationFailed')
 export type AuthenticationFailedErrorData = { username: string, identityProviderName: string }
 export type AuthenticationFailedError = MageError<typeof ErrAuthenticationFailed, AuthenticationFailedErrorData>
diff --git a/service/src/ingress/ingress.app.impl.ts b/service/src/ingress/ingress.app.impl.ts
index af325c692..b462f67e5 100644
--- a/service/src/ingress/ingress.app.impl.ts
+++ b/service/src/ingress/ingress.app.impl.ts
@@ -1,24 +1,24 @@
 import { entityNotFound, infrastructureError } from '../app.api/app.api.errors'
 import { AppResponse } from '../app.api/app.api.global'
-import { MageEventId } from '../entities/events/entities.events'
-import { Team, TeamId } from '../entities/teams/entities.teams'
-import { User, UserId, UserRepository, UserRepositoryError } from '../entities/users/entities.users'
+import { UserRepository } from '../entities/users/entities.users'
 import { AdmitFromIdentityProviderOperation, AdmitFromIdentityProviderRequest, authenticationFailedError, EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
-import { createEnrollmentCandidateUser, IdentityProvider, IdentityProviderRepository, IdentityProviderUser, UserIngressBindingRepository, UserIngressBindings } from './ingress.entities'
+import { IdentityProviderRepository, IdentityProviderUser, UserIngressBindingRepository } from './ingress.entities'
+import { ProcessNewUserEnrollment } from './ingress.services.api'
 import { LocalIdpCreateAccountOperation } from './local-idp.app.api'
 import { JWTService, TokenAssertion } from './verification'
 
 
-export function CreateEnrollMyselfOperation(createLocalIdpAccount: LocalIdpCreateAccountOperation, idpRepo: IdentityProviderRepository, userRepo: UserRepository): EnrollMyselfOperation {
+export function CreateEnrollMyselfOperation(createLocalIdpAccount: LocalIdpCreateAccountOperation, idpRepo: IdentityProviderRepository, enrollNewUser: ProcessNewUserEnrollment): EnrollMyselfOperation {
   return async function enrollMyself(req: EnrollMyselfRequest): ReturnType<EnrollMyselfOperation> {
     const localAccountCreate = await createLocalIdpAccount(req)
     if (localAccountCreate.error) {
       return AppResponse.error(localAccountCreate.error)
     }
     const localAccount = localAccountCreate.success!
-    const candidateMageAccount: Partial<User> = {
+    const candidateMageAccount: IdentityProviderUser = {
       username: localAccount.username,
       displayName: req.displayName,
+      phones: [],
     }
     if (req.email) {
       candidateMageAccount.email = req.email
@@ -27,69 +27,17 @@ export function CreateEnrollMyselfOperation(createLocalIdpAccount: LocalIdpCreat
       candidateMageAccount.phones = [ { number: req.phone, type: 'Main' } ]
     }
     const localIdp = await idpRepo.findIdpByName('local')
+    if (!localIdp) {
+      throw new Error('local idp not found')
+    }
+    const enrollmentResult = await enrollNewUser(candidateMageAccount, localIdp)
+
     // TODO: auto-activate account after enrollment policy
     throw new Error('unimplemented')
   }
 }
 
-export interface AssignTeamMember {
-  (member: UserId, team: TeamId): Promise<boolean>
-}
-
-export interface FindEventTeam {
-  (mageEventId: MageEventId): Promise<Team | null>
-}
-
-async function enrollNewUser(idpAccount: IdentityProviderUser, idp: IdentityProvider, userRepo: UserRepository, ingressBindingRepo: UserIngressBindingRepository, findEventTeam: FindEventTeam, assignTeamMember: AssignTeamMember): Promise<{ mageAccount: User, ingressBindings: UserIngressBindings }> {
-  console.info(`enrolling new user account ${idpAccount.username} from identity provider ${idp.name}`)
-  const candidate = createEnrollmentCandidateUser(idpAccount, idp)
-  const mageAccount = await userRepo.create(candidate)
-  if (mageAccount instanceof UserRepositoryError) {
-    throw mageAccount
-  }
-  const ingressBindings = await ingressBindingRepo.saveUserIngressBinding(
-    mageAccount.id,
-    {
-      userId: mageAccount.id,
-      idpId: idp.id,
-      idpAccountId: idpAccount.username,
-      idpAccountAttrs: {},
-      // TODO: these do not have functionality yet
-      verified: true,
-      enabled: true,
-    }
-  )
-  if (ingressBindings instanceof Error) {
-    throw ingressBindings
-  }
-  const { assignToTeams, assignToEvents } = idp.userEnrollmentPolicy
-  const assignEnrolledToTeam = (teamId: TeamId): Promise<{ teamId: TeamId, assigned: boolean }> => {
-    return assignTeamMember(mageAccount.id, teamId)
-      .then(assigned => ({ teamId, assigned }))
-      .catch(err => {
-        console.error(`error assigning enrolled user ${mageAccount.username} to team ${teamId}`, err)
-        return { teamId, assigned: false }
-      })
-  }
-  const assignEnrolledToEventTeam = (eventId: MageEventId): Promise<{ eventId: MageEventId, teamId: TeamId | null, assigned: boolean }> => {
-    return findEventTeam(eventId)
-      .then<{ eventId: MageEventId, teamId: TeamId | null, assigned: boolean }>(eventTeam => {
-        if (eventTeam) {
-          return assignEnrolledToTeam(eventTeam.id).then(teamAssignment => ({ eventId, ...teamAssignment }))
-        }
-        console.error(`failed to find implicit team for event ${eventId} while enrolling user ${mageAccount.username}`)
-        return { eventId, teamId: null, assigned: false }
-      })
-      .catch(err => {
-        console.error(`error looking up implicit team for event ${eventId} while enrolling user ${mageAccount.username}`, err)
-        return { eventId, teamId: null, assigned: false }
-      })
-  }
-  await Promise.all([ ...assignToTeams.map(assignEnrolledToTeam), ...assignToEvents.map(assignEnrolledToEventTeam) ])
-  return { mageAccount, ingressBindings }
-}
-
-export function CreateAdmitFromIdentityProviderOperation(idpRepo: IdentityProviderRepository, ingressBindingRepo: UserIngressBindingRepository, userRepo: UserRepository, findEventTeam: FindEventTeam, assignTeamMember: AssignTeamMember, tokenService: JWTService): AdmitFromIdentityProviderOperation {
+export function CreateAdmitFromIdentityProviderOperation(idpRepo: IdentityProviderRepository, ingressBindingRepo: UserIngressBindingRepository, userRepo: UserRepository, enrollNewUser: ProcessNewUserEnrollment, tokenService: JWTService): AdmitFromIdentityProviderOperation {
   return async function admitFromIdentityProvider(req: AdmitFromIdentityProviderRequest): ReturnType<AdmitFromIdentityProviderOperation> {
     const idp = await idpRepo.findIdpByName(req.identityProviderName)
     if (!idp) {
@@ -104,7 +52,7 @@ export function CreateAdmitFromIdentityProviderOperation(idpRepo: IdentityProvid
             return { mageAccount: existingAccount, ingressBindings }
           })
         }
-        return enrollNewUser(idpAccount, idp, userRepo, ingressBindingRepo, findEventTeam, assignTeamMember)
+        return enrollNewUser(idpAccount, idp)
       })
       .then(enrolled => {
         const { mageAccount, ingressBindings } = enrolled
diff --git a/service/src/ingress/ingress.services.api.ts b/service/src/ingress/ingress.services.api.ts
new file mode 100644
index 000000000..7b80f98d3
--- /dev/null
+++ b/service/src/ingress/ingress.services.api.ts
@@ -0,0 +1,6 @@
+import { User } from '../entities/users/entities.users'
+import { IdentityProvider, IdentityProviderUser, UserIngressBindings } from './ingress.entities'
+
+export interface ProcessNewUserEnrollment {
+  (idpAccount: IdentityProviderUser, idp: IdentityProvider): Promise<{ mageAccount: User, ingressBindings: UserIngressBindings }>
+}
\ No newline at end of file
diff --git a/service/src/ingress/ingress.services.impl.ts b/service/src/ingress/ingress.services.impl.ts
new file mode 100644
index 000000000..434cb413d
--- /dev/null
+++ b/service/src/ingress/ingress.services.impl.ts
@@ -0,0 +1,64 @@
+import { MageEventId } from '../entities/events/entities.events'
+import { Team, TeamId } from '../entities/teams/entities.teams'
+import { User, UserId, UserRepository, UserRepositoryError } from '../entities/users/entities.users'
+import { createEnrollmentCandidateUser, IdentityProvider, IdentityProviderUser, UserIngressBindingRepository, UserIngressBindings } from './ingress.entities'
+import { ProcessNewUserEnrollment } from './ingress.services.api'
+
+export interface AssignTeamMember {
+  (member: UserId, team: TeamId): Promise<boolean>
+}
+
+export interface FindEventTeam {
+  (mageEventId: MageEventId): Promise<Team | null>
+}
+
+export function CreateProcessNewUserEnrollmentService(userRepo: UserRepository, ingressBindingRepo: UserIngressBindingRepository, findEventTeam: FindEventTeam, assignTeamMember: AssignTeamMember): ProcessNewUserEnrollment {
+  return async function processNewUserEnrollment(idpAccount: IdentityProviderUser, idp: IdentityProvider): Promise<{ mageAccount: User, ingressBindings: UserIngressBindings }> {
+    console.info(`enrolling new user account ${idpAccount.username} from identity provider ${idp.name}`)
+    const candidate = createEnrollmentCandidateUser(idpAccount, idp)
+    const mageAccount = await userRepo.create(candidate)
+    if (mageAccount instanceof UserRepositoryError) {
+      throw mageAccount
+    }
+    const ingressBindings = await ingressBindingRepo.saveUserIngressBinding(
+      mageAccount.id,
+      {
+        userId: mageAccount.id,
+        idpId: idp.id,
+        idpAccountId: idpAccount.username,
+        idpAccountAttrs: {},
+        // TODO: these do not have functionality yet
+        verified: true,
+        enabled: true,
+      }
+    )
+    if (ingressBindings instanceof Error) {
+      throw ingressBindings
+    }
+    const { assignToTeams, assignToEvents } = idp.userEnrollmentPolicy
+    const assignEnrolledToTeam = (teamId: TeamId): Promise<{ teamId: TeamId, assigned: boolean }> => {
+      return assignTeamMember(mageAccount.id, teamId)
+        .then(assigned => ({ teamId, assigned }))
+        .catch(err => {
+          console.error(`error assigning enrolled user ${mageAccount.username} to team ${teamId}`, err)
+          return { teamId, assigned: false }
+        })
+    }
+    const assignEnrolledToEventTeam = (eventId: MageEventId): Promise<{ eventId: MageEventId, teamId: TeamId | null, assigned: boolean }> => {
+      return findEventTeam(eventId)
+        .then<{ eventId: MageEventId, teamId: TeamId | null, assigned: boolean }>(eventTeam => {
+          if (eventTeam) {
+            return assignEnrolledToTeam(eventTeam.id).then(teamAssignment => ({ eventId, ...teamAssignment }))
+          }
+          console.error(`failed to find implicit team for event ${eventId} while enrolling user ${mageAccount.username}`)
+          return { eventId, teamId: null, assigned: false }
+        })
+        .catch(err => {
+          console.error(`error looking up implicit team for event ${eventId} while enrolling user ${mageAccount.username}`, err)
+          return { eventId, teamId: null, assigned: false }
+        })
+    }
+    await Promise.all([ ...assignToTeams.map(assignEnrolledToTeam), ...assignToEvents.map(assignEnrolledToEventTeam) ])
+    return { mageAccount, ingressBindings }
+  }
+}
\ No newline at end of file

From 2eb95277ab51a997e974c8f403f83ea88285bdd1 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 2 Nov 2024 19:04:20 -0600
Subject: [PATCH 133/183] refactor(service): users/auth: better name for paging
 links function

---
 service/src/entities/entities.global.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/service/src/entities/entities.global.ts b/service/src/entities/entities.global.ts
index af69592f3..e728b6238 100644
--- a/service/src/entities/entities.global.ts
+++ b/service/src/entities/entities.global.ts
@@ -91,7 +91,7 @@ export interface Links {
   prev: number | null
 }
 
-export function calculateLinks(paging: PagingParameters, totalCount: number | null): Links {
+export function calculatePagingLinks(paging: PagingParameters, totalCount: number | null): Links {
   const limit = paging.pageSize
   const start = paging.pageIndex * limit
   const next = start + paging.pageSize < (totalCount || 0) ? paging.pageIndex + 1 : null
@@ -101,7 +101,7 @@ export function calculateLinks(paging: PagingParameters, totalCount: number | nu
 
 export const pageOf = <T>(items: T[], paging: PagingParameters, totalCount?: number | null): PageOf<T> => {
   const resolvedTotalCount = totalCount || 0;
-  const links = calculateLinks(paging, resolvedTotalCount)
+  const links = calculatePagingLinks(paging, resolvedTotalCount)
   return {
     totalCount: totalCount || null,
     pageSize: typeof paging.pageSize === 'number' ? paging.pageSize : -1,

From 500ff6dd8a34ce765b62cb14eb4a0ad852f63e31 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 4 Nov 2024 12:14:20 -0700
Subject: [PATCH 134/183] refactor(service): users/auth: add some functionality
 to user ingress bindings entity and repository

---
 ...identity-providers.adapters.db.mongoose.ts | 78 +++++++++++++++----
 service/src/ingress/ingress.app.impl.ts       | 10 +--
 service/src/ingress/ingress.entities.ts       | 36 +++++++--
 service/src/ingress/ingress.services.api.ts   |  2 +-
 service/src/ingress/ingress.services.impl.ts  | 10 +--
 5 files changed, 101 insertions(+), 35 deletions(-)

diff --git a/service/src/ingress/identity-providers.adapters.db.mongoose.ts b/service/src/ingress/identity-providers.adapters.db.mongoose.ts
index 1be0fae37..2eae1bcec 100644
--- a/service/src/ingress/identity-providers.adapters.db.mongoose.ts
+++ b/service/src/ingress/identity-providers.adapters.db.mongoose.ts
@@ -1,20 +1,13 @@
 import mongoose from 'mongoose'
 import { BaseMongooseRepository } from '../adapters/base/adapters.base.db.mongoose'
-import { MageEventId } from '../entities/events/entities.events'
-import { TeamId } from '../entities/teams/entities.teams'
-import { DeviceEnrollmentPolicy, IdentityProvider, IdentityProviderRepository, UserEnrollmentPolicy } from './ingress.entities'
+import { PagingParameters, PageOf } from '../entities/entities.global'
+import { UserId } from '../entities/users/entities.users'
+import { DeviceEnrollmentPolicy, IdentityProvider, IdentityProviderId, IdentityProviderMutableAttrs, IdentityProviderRepository, UserEnrollmentPolicy, UserIngressBinding, UserIngressBindings, UserIngressBindingsRepository } from './ingress.entities'
 
 type ObjectId = mongoose.Types.ObjectId
-
+const ObjectId = mongoose.Types.ObjectId
 const Schema = mongoose.Schema
 
-export type CommonIdpSettings = {
-  usersReqAdmin?: { enabled: boolean }
-  devicesReqAdmin?: { enabled: boolean }
-  newUserTeams?: TeamId[]
-  newUserEvents?: MageEventId[]
-}
-
 export type IdentityProviderDocument = Omit<IdentityProvider, 'id'> & {
   _id: ObjectId
 }
@@ -127,22 +120,75 @@ export class IdentityProviderMongooseRepository extends BaseMongooseRepository<I
   }
 
   async findIdpByName(name: string): Promise<IdentityProvider | null> {
-    const doc = await this.model.findOne({ name })
+    const doc = await this.model.findOne({ name }, null, { lean: true })
     if (doc) {
       return this.entityForDocument(doc)
     }
     return null
   }
 
-  updateIdp(update: Partial<IdentityProvider> & Pick<IdentityProvider, 'id'>): Promise<IdentityProvider | null> {
-    throw new Error('Method not implemented.')
+  updateIdp(update: Partial<IdentityProviderMutableAttrs> & Pick<IdentityProvider, 'id'>): Promise<IdentityProvider | null> {
+    return super.update(update)
   }
 
-  deleteIdp(id: string): Promise<IdentityProvider | null> {
+  deleteIdp(id: IdentityProviderId): Promise<IdentityProvider | null> {
     return super.removeById(id)
   }
 }
 
+
+export type UserIngressBindingsDocument = {
+  /**
+   * The ingress bindings `_id` is actually the `_id` of the related user document.
+   */
+  _id: ObjectId
+  bindings: { [idpId: IdentityProviderId]: UserIngressBinding }
+}
+
+export type UserIngressBindingsModel = mongoose.Model<UserIngressBindingsDocument>
+
+export const UserIngressBindingsSchema = new Schema<UserIngressBindingsDocument>(
+  {
+    bindings: { type: Schema.Types.Mixed, required: true }
+  }
+)
+
+export class UserIngressBindingsMongooseRepository implements UserIngressBindingsRepository {
+
+  constructor(readonly model: UserIngressBindingsModel) {}
+
+  async readBindingsForUser(userId: UserId): Promise<UserIngressBindings> {
+    const doc = await this.model.findById(userId, null, { lean: true })
+    return { userId, bindings: new Map(Object.entries(doc?.bindings || {})) }
+  }
+
+  async readAllBindingsForIdp(idpId: IdentityProviderId, paging?: PagingParameters | undefined): Promise<PageOf<UserIngressBindings>> {
+    throw new Error('Method not implemented.')
+  }
+
+  async saveUserIngressBinding(userId: UserId, binding: UserIngressBinding): Promise<Error | UserIngressBindings> {
+    const _id = new ObjectId(userId)
+    const bindingsUpdate = { $set: { [`bindings.${binding.idpId}`]: binding } }
+    const doc = await this.model.findOneAndUpdate({ _id }, bindingsUpdate, { upsert: true, new: true })
+    return { userId, bindings: new Map(Object.entries(doc.bindings)) }
+  }
+
+  async deleteBinding(userId: UserId, idpId: IdentityProviderId): Promise<UserIngressBinding | null> {
+    const _id = new ObjectId(userId)
+    const bindingsUpdate = { $unset: [`bindings.${idpId}`] }
+    const doc = await this.model.findOneAndUpdate({ _id }, bindingsUpdate)
+    return doc?.bindings[idpId] || null
+  }
+
+  async deleteBindingsForUser(userId: UserId): Promise<UserIngressBindings | null> {
+    throw new Error('Method not implemented.')
+  }
+
+  async deleteAllBindingsForIdp(idpId: IdentityProviderId): Promise<number> {
+    throw new Error('Method not implemented.')
+  }
+}
+
 // TODO: should be per protocol and identity provider and in entity layer
 const whitelist = ['name', 'type', 'title', 'textColor', 'buttonColor', 'icon'];
 const blacklist = ['clientsecret', 'bindcredentials', 'privatecert', 'decryptionpvk'];
@@ -174,7 +220,7 @@ function DbAuthenticationConfigurationToObject(config, ret, options) {
   ret.icon = ret.icon ? ret.icon.toString('base64') : null;
 }
 
-// TODO: move to api layer
+// TODO: move to api/web layer
 function manageIcon(config) {
   if (config.icon) {
     if (config.icon.startsWith('data')) {
diff --git a/service/src/ingress/ingress.app.impl.ts b/service/src/ingress/ingress.app.impl.ts
index b462f67e5..ae46cd297 100644
--- a/service/src/ingress/ingress.app.impl.ts
+++ b/service/src/ingress/ingress.app.impl.ts
@@ -2,13 +2,13 @@ import { entityNotFound, infrastructureError } from '../app.api/app.api.errors'
 import { AppResponse } from '../app.api/app.api.global'
 import { UserRepository } from '../entities/users/entities.users'
 import { AdmitFromIdentityProviderOperation, AdmitFromIdentityProviderRequest, authenticationFailedError, EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
-import { IdentityProviderRepository, IdentityProviderUser, UserIngressBindingRepository } from './ingress.entities'
-import { ProcessNewUserEnrollment } from './ingress.services.api'
+import { IdentityProviderRepository, IdentityProviderUser, UserIngressBindingsRepository } from './ingress.entities'
+import { EnrollNewUser } from './ingress.services.api'
 import { LocalIdpCreateAccountOperation } from './local-idp.app.api'
 import { JWTService, TokenAssertion } from './verification'
 
 
-export function CreateEnrollMyselfOperation(createLocalIdpAccount: LocalIdpCreateAccountOperation, idpRepo: IdentityProviderRepository, enrollNewUser: ProcessNewUserEnrollment): EnrollMyselfOperation {
+export function CreateEnrollMyselfOperation(createLocalIdpAccount: LocalIdpCreateAccountOperation, idpRepo: IdentityProviderRepository, enrollNewUser: EnrollNewUser): EnrollMyselfOperation {
   return async function enrollMyself(req: EnrollMyselfRequest): ReturnType<EnrollMyselfOperation> {
     const localAccountCreate = await createLocalIdpAccount(req)
     if (localAccountCreate.error) {
@@ -37,7 +37,7 @@ export function CreateEnrollMyselfOperation(createLocalIdpAccount: LocalIdpCreat
   }
 }
 
-export function CreateAdmitFromIdentityProviderOperation(idpRepo: IdentityProviderRepository, ingressBindingRepo: UserIngressBindingRepository, userRepo: UserRepository, enrollNewUser: ProcessNewUserEnrollment, tokenService: JWTService): AdmitFromIdentityProviderOperation {
+export function CreateAdmitFromIdentityProviderOperation(idpRepo: IdentityProviderRepository, ingressBindingRepo: UserIngressBindingsRepository, userRepo: UserRepository, enrollNewUser: EnrollNewUser, tokenService: JWTService): AdmitFromIdentityProviderOperation {
   return async function admitFromIdentityProvider(req: AdmitFromIdentityProviderRequest): ReturnType<AdmitFromIdentityProviderOperation> {
     const idp = await idpRepo.findIdpByName(req.identityProviderName)
     if (!idp) {
@@ -56,7 +56,7 @@ export function CreateAdmitFromIdentityProviderOperation(idpRepo: IdentityProvid
       })
       .then(enrolled => {
         const { mageAccount, ingressBindings } = enrolled
-        if (ingressBindings.has(idp.id)) {
+        if (ingressBindings.bindingsByIdp.has(idp.id)) {
           return mageAccount
         }
         console.error(`user ${mageAccount.username} has no ingress binding to identity provider ${idp.name}`)
diff --git a/service/src/ingress/ingress.entities.ts b/service/src/ingress/ingress.entities.ts
index 9c6f27f4d..dadf3b6c7 100644
--- a/service/src/ingress/ingress.entities.ts
+++ b/service/src/ingress/ingress.entities.ts
@@ -4,6 +4,7 @@ import { MageEventId } from '../entities/events/entities.events'
 import { TeamId } from '../entities/teams/entities.teams'
 import { UserExpanded, UserId } from '../entities/users/entities.users'
 import { RoleId } from '../entities/authorization/entities.authorization'
+import { PageOf, PagingParameters } from '../entities/entities.global'
 
 export interface Session {
   token: string
@@ -88,14 +89,27 @@ export type IdentityProviderUser = Pick<User, 'username' | 'displayName' | 'emai
  * identity provider for Mage to map the identity provider account to the Mage user account.
  */
 export interface UserIngressBinding {
-  userId: UserId
   idpId: IdentityProviderId
-  verified: boolean
-  enabled: boolean
+  // TODO: evaluate for utility of disabling a single ingress/idp path for a user as opposed to the entire account
+  // verified: boolean
+  // enabled: boolean
+  /**
+   * The identity provider account ID is the identifier of the account native to the owning identity provider.  This is
+   * only necessary if the identifier differs from the Mage account's username, especially if a Mage account has
+   * multiple ingress bindings for different identity providers with different account identifiers.
+   */
   idpAccountId?: string
+  /**
+   * Any attributes the identity provider or protocol needs to persist about the account mapping
+   */
   idpAccountAttrs?: Record<string, any>
 }
 
+export type UserIngressBindings = {
+  userId: UserId
+  bindingsByIdp: Map<IdentityProviderId, UserIngressBinding>
+}
+
 export type IdentityProviderMutableAttrs = Omit<IdentityProvider, 'id' | 'name' | 'protocol'>
 
 export interface IdentityProviderRepository {
@@ -109,12 +123,22 @@ export interface IdentityProviderRepository {
   deleteIdp(id: IdentityProviderId): Promise<IdentityProvider | null>
 }
 
-export type UserIngressBindings = Map<IdentityProviderId, UserIngressBinding>
-
-export interface UserIngressBindingRepository {
+export interface UserIngressBindingsRepository {
   readBindingsForUser(userId: UserId): Promise<UserIngressBindings>
+  readAllBindingsForIdp(idpId: IdentityProviderId, paging?: PagingParameters): Promise<PageOf<UserIngressBindings>>
   saveUserIngressBinding(userId: UserId, binding: UserIngressBinding): Promise<UserIngressBindings | Error>
+  /**
+   * Return the binding that was deleted, or null if the user did not have a binding to the given IDP.
+   */
   deleteBinding(userId: UserId, idpId: IdentityProviderId): Promise<UserIngressBinding | null>
+  /**
+   * Return the bindings that were deleted for the given user, or null if the user had no ingress bindings.
+   */
+  deleteBindingsForUser(userId: UserId): Promise<UserIngressBindings | null>
+  /**
+   * Return the number of deleted bindings.
+   */
+  deleteAllBindingsForIdp(idpId: IdentityProviderId): Promise<number>
 }
 
 /**
diff --git a/service/src/ingress/ingress.services.api.ts b/service/src/ingress/ingress.services.api.ts
index 7b80f98d3..63315f1c0 100644
--- a/service/src/ingress/ingress.services.api.ts
+++ b/service/src/ingress/ingress.services.api.ts
@@ -1,6 +1,6 @@
 import { User } from '../entities/users/entities.users'
 import { IdentityProvider, IdentityProviderUser, UserIngressBindings } from './ingress.entities'
 
-export interface ProcessNewUserEnrollment {
+export interface EnrollNewUser {
   (idpAccount: IdentityProviderUser, idp: IdentityProvider): Promise<{ mageAccount: User, ingressBindings: UserIngressBindings }>
 }
\ No newline at end of file
diff --git a/service/src/ingress/ingress.services.impl.ts b/service/src/ingress/ingress.services.impl.ts
index 434cb413d..540aec846 100644
--- a/service/src/ingress/ingress.services.impl.ts
+++ b/service/src/ingress/ingress.services.impl.ts
@@ -1,8 +1,8 @@
 import { MageEventId } from '../entities/events/entities.events'
 import { Team, TeamId } from '../entities/teams/entities.teams'
 import { User, UserId, UserRepository, UserRepositoryError } from '../entities/users/entities.users'
-import { createEnrollmentCandidateUser, IdentityProvider, IdentityProviderUser, UserIngressBindingRepository, UserIngressBindings } from './ingress.entities'
-import { ProcessNewUserEnrollment } from './ingress.services.api'
+import { createEnrollmentCandidateUser, IdentityProvider, IdentityProviderUser, UserIngressBindingsRepository, UserIngressBindings } from './ingress.entities'
+import { EnrollNewUser } from './ingress.services.api'
 
 export interface AssignTeamMember {
   (member: UserId, team: TeamId): Promise<boolean>
@@ -12,7 +12,7 @@ export interface FindEventTeam {
   (mageEventId: MageEventId): Promise<Team | null>
 }
 
-export function CreateProcessNewUserEnrollmentService(userRepo: UserRepository, ingressBindingRepo: UserIngressBindingRepository, findEventTeam: FindEventTeam, assignTeamMember: AssignTeamMember): ProcessNewUserEnrollment {
+export function CreateProcessNewUserEnrollmentService(userRepo: UserRepository, ingressBindingRepo: UserIngressBindingsRepository, findEventTeam: FindEventTeam, assignTeamMember: AssignTeamMember): EnrollNewUser {
   return async function processNewUserEnrollment(idpAccount: IdentityProviderUser, idp: IdentityProvider): Promise<{ mageAccount: User, ingressBindings: UserIngressBindings }> {
     console.info(`enrolling new user account ${idpAccount.username} from identity provider ${idp.name}`)
     const candidate = createEnrollmentCandidateUser(idpAccount, idp)
@@ -23,13 +23,9 @@ export function CreateProcessNewUserEnrollmentService(userRepo: UserRepository,
     const ingressBindings = await ingressBindingRepo.saveUserIngressBinding(
       mageAccount.id,
       {
-        userId: mageAccount.id,
         idpId: idp.id,
         idpAccountId: idpAccount.username,
         idpAccountAttrs: {},
-        // TODO: these do not have functionality yet
-        verified: true,
-        enabled: true,
       }
     )
     if (ingressBindings instanceof Error) {

From 1ce7295230062b5a5389214f3bb79791325352b0 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 4 Nov 2024 12:25:37 -0700
Subject: [PATCH 135/183] refactor(service): users/auth: remove unused ingress
 db adapter module

---
 .../ingress/ingress.adapters.db.mongoose.ts   | 32 -------------------
 1 file changed, 32 deletions(-)
 delete mode 100644 service/src/ingress/ingress.adapters.db.mongoose.ts

diff --git a/service/src/ingress/ingress.adapters.db.mongoose.ts b/service/src/ingress/ingress.adapters.db.mongoose.ts
deleted file mode 100644
index b5ca2860f..000000000
--- a/service/src/ingress/ingress.adapters.db.mongoose.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-import mongoose from 'mongoose'
-
-type ObjectId = mongoose.Types.ObjectId
-const Schema = mongoose.Schema
-
-const UserIngressSchema = new Schema(
-  {
-    // TODO: type is really not necessary
-    type: { type: String, required: true },
-    id: { type: String, required: false },
-    authenticationConfigurationId: { type: Schema.Types.ObjectId, ref: 'AuthenticationConfiguration', required: false }
-  },
-  {
-    timestamps: {
-      updatedAt: 'lastUpdated'
-    },
-    toObject: {
-      transform: DbAuthenticationToObject
-    }
-  }
-);
-
-export type UserIdpAccountDocument = {
-  _id: ObjectId
-  // TODO: migrate to this foreign key instead of on user records
-  // userId: ObjectId
-  createdAt: Date
-  lastUpdated: Date
-  // TODO: migrate to identityProviderId
-  authenticationConfigurationId: ObjectId
-  idpAccount: Record<string, any>
-}
\ No newline at end of file

From ff7ec537c5154a6cf4d629ecf795ddaa74fbc489 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 4 Nov 2024 12:26:17 -0700
Subject: [PATCH 136/183] refactor(service): users/auth: fix references to
 refactored names

---
 .../src/ingress/identity-providers.adapters.db.mongoose.ts    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/service/src/ingress/identity-providers.adapters.db.mongoose.ts b/service/src/ingress/identity-providers.adapters.db.mongoose.ts
index 2eae1bcec..b31d97e98 100644
--- a/service/src/ingress/identity-providers.adapters.db.mongoose.ts
+++ b/service/src/ingress/identity-providers.adapters.db.mongoose.ts
@@ -159,7 +159,7 @@ export class UserIngressBindingsMongooseRepository implements UserIngressBinding
 
   async readBindingsForUser(userId: UserId): Promise<UserIngressBindings> {
     const doc = await this.model.findById(userId, null, { lean: true })
-    return { userId, bindings: new Map(Object.entries(doc?.bindings || {})) }
+    return { userId, bindingsByIdp: new Map(Object.entries(doc?.bindings || {})) }
   }
 
   async readAllBindingsForIdp(idpId: IdentityProviderId, paging?: PagingParameters | undefined): Promise<PageOf<UserIngressBindings>> {
@@ -170,7 +170,7 @@ export class UserIngressBindingsMongooseRepository implements UserIngressBinding
     const _id = new ObjectId(userId)
     const bindingsUpdate = { $set: { [`bindings.${binding.idpId}`]: binding } }
     const doc = await this.model.findOneAndUpdate({ _id }, bindingsUpdate, { upsert: true, new: true })
-    return { userId, bindings: new Map(Object.entries(doc.bindings)) }
+    return { userId, bindingsByIdp: new Map(Object.entries(doc.bindings)) }
   }
 
   async deleteBinding(userId: UserId, idpId: IdentityProviderId): Promise<UserIngressBinding | null> {

From 1cbd6fc63d355a2afa0ce5b84a5f39c7fd073a8a Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 5 Nov 2024 06:19:27 -0700
Subject: [PATCH 137/183] refactor(service): users/auth: rename identity
 providers db adapter module to ingress because it will contain more than just
 idps

---
 ...rs.adapters.db.mongoose.ts => ingress.adapters.db.mongoose.ts} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/src/ingress/{identity-providers.adapters.db.mongoose.ts => ingress.adapters.db.mongoose.ts} (100%)

diff --git a/service/src/ingress/identity-providers.adapters.db.mongoose.ts b/service/src/ingress/ingress.adapters.db.mongoose.ts
similarity index 100%
rename from service/src/ingress/identity-providers.adapters.db.mongoose.ts
rename to service/src/ingress/ingress.adapters.db.mongoose.ts

From b98e722aebb05464cf6c1371cf13d5b4b7afa7e7 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 5 Nov 2024 06:23:26 -0700
Subject: [PATCH 138/183] refactor(service): users/auth: rename session
 repository factory function for consistency

---
 service/src/ingress/sessions.adapters.db.mongoose.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/ingress/sessions.adapters.db.mongoose.ts b/service/src/ingress/sessions.adapters.db.mongoose.ts
index debf13278..69523e59e 100644
--- a/service/src/ingress/sessions.adapters.db.mongoose.ts
+++ b/service/src/ingress/sessions.adapters.db.mongoose.ts
@@ -33,7 +33,7 @@ const populateSessionUserRole: mongoose.PopulateOptions = {
   populate: 'roleId'
 }
 
-export function createSessionRepository(conn: mongoose.Connection, collectionName: string, sessionTimeoutSeconds: number): SessionRepository {
+export function SessionsMongooseRepository(conn: mongoose.Connection, collectionName: string, sessionTimeoutSeconds: number): SessionRepository {
   const model = conn.model('Token', SessionSchema, collectionName)
   return Object.freeze({
     model,

From 70999cdd200eb3b1f1b68a88b6f9cf2049192021 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 5 Nov 2024 12:07:39 -0700
Subject: [PATCH 139/183] refactor(service): users/auth: remove passport bearer
 middleware from event routes

---
 service/src/routes/events.ts | 46 +++++++-----------------------------
 1 file changed, 9 insertions(+), 37 deletions(-)

diff --git a/service/src/routes/events.ts b/service/src/routes/events.ts
index 1b5b1e3fe..9476f9fd4 100644
--- a/service/src/routes/events.ts
+++ b/service/src/routes/events.ts
@@ -1,15 +1,14 @@
 const api = require('../api')
-const userTransformer = require('../transformers/user')
 
 import async from 'async'
 import util from 'util'
 import fileType from 'file-type'
+import mongoose from 'mongoose'
 import EventModel, { FormDocument, MageEventDocument } from '../models/event'
 import express from 'express'
 import access from '../access'
 import { AnyPermission, MageEventPermission } from '../entities/authorization/entities.permissions'
 import { JsonObject } from '../entities/entities.json_types'
-import authentication from '../authentication'
 import fs from 'fs-extra'
 import { EventAccessType, MageEvent } from '../entities/events/entities.events'
 import { defaultHandler as upload } from '../upload'
@@ -42,10 +41,10 @@ function determineReadAccess(req: express.Request, res: express.Response, next:
 function middlewareAuthorizeAccess(collectionPermission: AnyPermission, aclPermission: EventAccessType): express.RequestHandler {
   return async (req, res, next) => {
     const denied = await defaultEventPermissionsService.authorizeEventAccess(req.event!, req.user, collectionPermission, aclPermission)
-    if (!denied) {
-      return next()
+    if (denied) {
+      return res.sendStatus(403)
     }
-    return res.sendStatus(403)
+    next()
   }
 }
 
@@ -167,10 +166,10 @@ function reduceStyle(style: any): LineStyle {
 }
 
 
-
-function EventRoutes(app: express.Application, security: { authentication: authentication.AuthLayer }): void {
-
-  const passport = security.authentication.passport;
+/**
+ * TODO: users-next: replace old authentication reference
+ */
+function EventRoutes(app: express.Application): void {
 
   /*
   TODO: this just sends whatever is in the body straight through the API level
@@ -180,7 +179,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
   */
   app.post(
     '/api/events',
-    passport.authenticate('bearer'),
     access.authorize(MageEventPermission.CREATE_EVENT),
     function(req, res, next) {
       new api.Event().createEvent(req.body, req.user, function(err: any, event: MageEventDocument) {
@@ -194,7 +192,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.get(
     '/api/events',
-    passport.authenticate('bearer'),
     determineReadAccess,
     parseEventQueryParams,
     function (req, res, next) {
@@ -217,7 +214,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.get(
     '/api/events/count',
-    passport.authenticate('bearer'),
     determineReadAccess,
     function(req, res, next) {
       EventModel.count({access: req.access}, function(err, count) {
@@ -230,7 +226,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.get(
     '/api/events/:eventId',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.READ_EVENT_ALL, EventAccessType.Read),
     determineReadAccess,
     parseEventQueryParams,
@@ -248,7 +243,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.put(
     '/api/events/:eventId',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     function(req, res, next) {
       new api.Event(req.event).updateEvent(req.body, {}, function(err: any, event: MageEventDocument) {
@@ -267,7 +261,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.delete(
     '/api/events/:eventId',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.DELETE_EVENT, EventAccessType.Delete),
     function(req, res, next) {
       new api.Event(req.event).deleteEvent(function(err: any) {
@@ -279,7 +272,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.post(
     '/api/events/:eventId/forms',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     upload.single('form'),
     function(req, res, next) {
@@ -321,12 +313,11 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.post(
     '/api/events/:eventId/forms',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     parseForm,
     function(req, res, next) {
       const form = req.form;
-      new api.Event(req.event).addForm(form, function(err: any, form: FormDocument) {
+      new api.Event(req.event).addForm(form, function(err: any, form: mongoose.HydratedDocument<FormDocument>) {
         if (err) return next(err);
 
         async.parallel([
@@ -350,7 +341,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.put(
     '/api/events/:eventId/forms/:formId',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     parseForm,
     function(req, res, next) {
@@ -374,7 +364,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
   // TODO: why not /api/events/:eventId/forms/:formId.zip?
   app.get(
     '/api/events/:eventId/:formId/form.zip',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.READ_EVENT_ALL, EventAccessType.Read),
     function(req, res, next) {
       new api.Form(req.event).export(parseInt(req.params.formId, 10), function(err: any, form: any) {
@@ -389,7 +378,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.get(
     '/api/events/:eventId/form/icons.zip',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.READ_EVENT_ALL, EventAccessType.Read),
     function(req, res) {
       new api.Icon(req.event!._id).getZipPath(function(err: any, zipPath: string) {
@@ -407,7 +395,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
   // the eventId parameter is not even used.
   app.get(
     '/api/events/:eventId/form/icons*',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.READ_EVENT_ALL, EventAccessType.Read),
     function(req, res) {
       res.sendFile(api.Icon.defaultIconPath);
@@ -416,7 +403,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.get(
     '/api/events/:eventId/icons/:formId.json',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.READ_EVENT_ALL, EventAccessType.Read),
     function(req, res, next) {
       new api.Icon(req.event!._id, req.params.formId).getIcons(function(err: any, icons: any) {
@@ -457,7 +443,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
   // TODO: should be PUT?
   app.post(
     '/api/events/:eventId/icons/:formId?/:primary?/:variant?',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     upload.single('icon'),
     function(req, res, next) {
@@ -473,7 +458,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
   // get icon
   app.get(
     '/api/events/:eventId/icons/:formId?/:primary?/:variant?',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.READ_EVENT_ALL, EventAccessType.Read),
     function(req, res, next) {
       new api.Icon(req.event!._id, req.params.formId, req.params.primary, req.params.variant).getIcon(function(err: any, icon: any) {
@@ -507,7 +491,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
   // Delete an icon
   app.delete(
     '/api/events/:eventId/icons/:formId?/:primary?/:variant?',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     function(req, res, next) {
       new api.Icon(req.event!._id, req.params.formId, req.params.primary, req.params.variant).delete(function(err: any) {
@@ -520,7 +503,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.post(
     '/api/events/:eventId/layers',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     function(req, res, next) {
       EventModel.addLayer(req.event!, req.body, function(err: any, event?: MageEventDocument) {
@@ -534,7 +516,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.delete(
     '/api/events/:eventId/layers/:id',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     function(req, res, next) {
       EventModel.removeLayer(req.event!, {id: req.params.id}, function(err: any, event?: MageEventDocument) {
@@ -548,7 +529,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.get(
     '/api/events/:eventId/users',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.READ_EVENT_ALL, EventAccessType.Read),
     determineReadAccess,
     function (req, res, next) {
@@ -564,7 +544,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.post(
     '/api/events/:eventId/teams',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     function(req, res, next) {
       EventModel.addTeam(req.event!, req.body, function(err, event) {
@@ -578,7 +557,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.delete(
     '/api/events/:eventId/teams/:teamId',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     function(req, res, next) {
       EventModel.removeTeam(req.event!, req.team, function(err, event) {
@@ -592,7 +570,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.put(
     '/api/events/:eventId/acl/:targetUserId',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     function(req, res, next) {
       EventModel.updateUserInAcl(req.event!._id, req.params.targetUserId, req.body.role, function(err, event) {
@@ -606,7 +583,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.delete(
     '/api/events/:eventId/acl/:targetUserId',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.UPDATE_EVENT, EventAccessType.Update),
     function(req, res, next) {
       EventModel.removeUserFromAcl(req.event!._id, req.params.targetUserId, function(err, event) {
@@ -620,7 +596,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.get(
     '/api/events/:id/members',
-    passport.authenticate('bearer'),
     determineReadAccess,
     function (req, res, next) {
       const options = {
@@ -641,7 +616,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.get(
     '/api/events/:id/nonMembers',
-    passport.authenticate('bearer'),
     determineReadAccess,
     function (req, res, next) {
       const options = {
@@ -674,7 +648,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.get(
     '/api/events/:eventId/teams',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.READ_EVENT_ALL, EventAccessType.Read),
     function (req, res, next) {
       const options = teamQueryOptionsFromRequest(req)
@@ -694,7 +667,6 @@ function EventRoutes(app: express.Application, security: { authentication: authe
 
   app.get(
     '/api/events/:eventId/nonTeams',
-    passport.authenticate('bearer'),
     middlewareAuthorizeAccess(MageEventPermission.READ_EVENT_ALL, EventAccessType.Read),
     function (req, res, next) {
       const options = teamQueryOptionsFromRequest(req)

From 4e990694cbbdcdc24917fdd63ded349b8f57665a Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 5 Nov 2024 12:23:56 -0700
Subject: [PATCH 140/183] refactor(service): users/auth: remove user and role
 mongoose model references permission utility

---
 service/src/access/index.ts | 26 +++++++++-----------------
 1 file changed, 9 insertions(+), 17 deletions(-)

diff --git a/service/src/access/index.ts b/service/src/access/index.ts
index 2f660cc71..2dc265e69 100644
--- a/service/src/access/index.ts
+++ b/service/src/access/index.ts
@@ -4,8 +4,7 @@
 
 import express from 'express'
 import { AnyPermission } from '../entities/authorization/entities.permissions'
-import { RoleDocument } from '../models/role'
-import { UserDocument } from '../models/user'
+import { UserExpanded } from '../entities/users/entities.users'
 
 export = Object.freeze({
 
@@ -20,28 +19,21 @@ export = Object.freeze({
    */
   authorize(permission: AnyPermission): express.RequestHandler {
     return function(req, res, next): any {
-      if (!req.user) {
+      if (req.user?.from !== 'sessionToken') {
         return next()
       }
-      const role = req.user.roleId as RoleDocument
-      if (!role) {
-        return res.sendStatus(403)
-      }
+      const role = req.user.account.role
       const userPermissions = role.permissions
-      const ok = userPermissions.indexOf(permission) !== -1
-      if (!ok) {
-        return res.sendStatus(403)
+      if (userPermissions.includes(permission)) {
+        return next()
       }
-      next()
+      return res.sendStatus(403)
     }
   },
 
   // TODO: users-next
-  userHasPermission(user: UserDocument, permission: AnyPermission) {
-    if (!user || !user.roleId) {
-      return false
-    }
-    const role = user.roleId as RoleDocument
-    return role.permissions.indexOf(permission) !== -1
+  userHasPermission(user: UserExpanded | null | undefined, permission: AnyPermission): boolean {
+    const role = user?.role
+    return role ? role.permissions.indexOf(permission) !== -1 : false
   }
 })

From fa66fa541453fb18e0609e03a40e8f745da49fc7 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 5 Nov 2024 12:27:05 -0700
Subject: [PATCH 141/183] refactor(service): users/auth: add discriminated
 union for express passport user type to distinguish between session and idp
 users

---
 service/src/@types/express/index.d.ts | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/service/src/@types/express/index.d.ts b/service/src/@types/express/index.d.ts
index 8cb01edd7..41755dbe5 100644
--- a/service/src/@types/express/index.d.ts
+++ b/service/src/@types/express/index.d.ts
@@ -1,9 +1,24 @@
-import { UserDocument } from '../../models/user'
+import { UserExpanded as MageUser } from '../../entities/users/entities.users'
+import { IdentityProviderUser } from '../../ingress/ingress.entities'
+
+
+export type WebIngressUserFromSessionToken = { from: 'sessionToken', account: MageUser }
+export type WebIngressUserFromIdentityProvider = { from: 'identityProvider', account: IdentityProviderUser }
+/**
+ * The `WebIngressUser` type determines the ingress path of the requesting user through the Passport middleware stack.
+ * When the `mage` key is present, the requesting user authenticated using an established Mage session token.  When the
+ * `identityProvider` key is present, the requesting user authenticated with a third party identity provider account
+ * and will establish a new Mage session.
+ */
+export type WebIngressUser = WebIngressUserFromSessionToken | WebIngressUserFromIdentityProvider
 
 declare module 'express-serve-static-core' {
   export interface Request {
-    user: UserDocument & any
-    provisionedDeviceId: string
+    user?: Express.User & WebIngressUser
+    token?: string
+    // TODO: users-next: reconcile these two device properties and change to device entity
+    provisionedDevice?: any
+    provisionedDeviceId?: string
     /**
      * Return the root HTTP URL of the server, including the scheme, e.g.,
      * `https://mage.io`.

From 3fd21f596a8536b3f828364e49bc1e46788169df Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 5 Nov 2024 14:08:56 -0700
Subject: [PATCH 142/183] refactor(service): users/auth: event routes returns a
 new router instead of adding routes to express app

---
 service/src/routes/events.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/service/src/routes/events.ts b/service/src/routes/events.ts
index 9476f9fd4..f345364d8 100644
--- a/service/src/routes/events.ts
+++ b/service/src/routes/events.ts
@@ -169,7 +169,9 @@ function reduceStyle(style: any): LineStyle {
 /**
  * TODO: users-next: replace old authentication reference
  */
-function EventRoutes(app: express.Application): void {
+function EventRoutes(): express.Router {
+
+  const app = express.Router()
 
   /*
   TODO: this just sends whatever is in the body straight through the API level
@@ -678,6 +680,8 @@ function EventRoutes(app: express.Application): void {
       }).catch(err => next(err));
     }
   );
+
+  return app
 }
 
 export = EventRoutes

From 8fd977f7d218750443874e5934e9c72c81018336 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 5 Nov 2024 14:12:09 -0700
Subject: [PATCH 143/183] refactor(service): users/auth: move legacy events web
 routes to adapters to exclude from legacy route loading mechanism

---
 .../events/adapters.events.controllers.web.legacy.ts}             | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename service/src/{routes/events.ts => adapters/events/adapters.events.controllers.web.legacy.ts} (100%)

diff --git a/service/src/routes/events.ts b/service/src/adapters/events/adapters.events.controllers.web.legacy.ts
similarity index 100%
rename from service/src/routes/events.ts
rename to service/src/adapters/events/adapters.events.controllers.web.legacy.ts

From 8c109cbca0b61d7c166a083d7744c3f73198a090 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Tue, 5 Nov 2024 14:12:32 -0700
Subject: [PATCH 144/183] refactor(service): users/auth: delete old user
 transformer module

---
 service/src/transformers/user.js | 21 ---------------------
 1 file changed, 21 deletions(-)
 delete mode 100644 service/src/transformers/user.js

diff --git a/service/src/transformers/user.js b/service/src/transformers/user.js
deleted file mode 100644
index 499ba88d6..000000000
--- a/service/src/transformers/user.js
+++ /dev/null
@@ -1,21 +0,0 @@
-function transformUser(user, options) {
-  if (!user) {
-    return null;
-  }
-  return user.toObject ?
-    user.toObject({ path: options.path }) :
-    user;
-}
-
-function transformUsers(users, options) {
-  return users.map(function(user) {
-    return transformUser(user, options);
-  });
-}
-
-exports.transform = function(users, options) {
-  options = options || {};
-  return Array.isArray(users) ?
-    transformUsers(users, options) :
-    transformUser(users, options);
-};

From c3b4d2e52593fd304fab298f2d23e3a4bf692d96 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 7 Nov 2024 09:44:37 -0700
Subject: [PATCH 145/183] refactor(service): users/auth: pass only user id vs
 user instance to event model filtering

---
 .../adapters.events.controllers.web.legacy.ts | 27 +++++----
 service/src/models/event.d.ts                 |  6 +-
 service/src/models/event.js                   | 55 ++++++-------------
 3 files changed, 35 insertions(+), 53 deletions(-)

diff --git a/service/src/adapters/events/adapters.events.controllers.web.legacy.ts b/service/src/adapters/events/adapters.events.controllers.web.legacy.ts
index f345364d8..96846a04d 100644
--- a/service/src/adapters/events/adapters.events.controllers.web.legacy.ts
+++ b/service/src/adapters/events/adapters.events.controllers.web.legacy.ts
@@ -4,22 +4,23 @@ import async from 'async'
 import util from 'util'
 import fileType from 'file-type'
 import mongoose from 'mongoose'
-import EventModel, { FormDocument, MageEventDocument } from '../models/event'
+import EventModel, { FormDocument, MageEventDocument } from '../../models/event'
 import express from 'express'
-import access from '../access'
-import { AnyPermission, MageEventPermission } from '../entities/authorization/entities.permissions'
-import { JsonObject } from '../entities/entities.json_types'
+import access from '../../access'
+import { AnyPermission, MageEventPermission } from '../../entities/authorization/entities.permissions'
+import { JsonObject } from '../../entities/entities.json_types'
 import fs from 'fs-extra'
-import { EventAccessType, MageEvent } from '../entities/events/entities.events'
-import { defaultHandler as upload } from '../upload'
-import { defaultEventPermissionsService } from '../permissions/permissions.events'
-import { LineStyle, PagingParameters } from '../entities/entities.global'
+import { EventAccessType, MageEvent } from '../../entities/events/entities.events'
+import { defaultHandler as upload } from '../../upload'
+import { defaultEventPermissionsService } from '../../permissions/permissions.events'
+import { LineStyle, PagingParameters } from '../../entities/entities.global'
+import { UserId } from '../../entities/users/entities.users'
 
 declare module 'express-serve-static-core' {
   export interface Request {
     event?: EventModel.MageEventDocument
     eventEntity?: MageEvent
-    access?: { user: express.Request['user'], permission: EventAccessType }
+    access?: { userId: UserId, permission: EventAccessType }
     parameters?: EventQueryParams
     form?: FormJson
     team?: any
@@ -27,10 +28,11 @@ declare module 'express-serve-static-core' {
 }
 
 function determineReadAccess(req: express.Request, res: express.Response, next: express.NextFunction): void {
-  if (!access.userHasPermission(req.user, MageEventPermission.READ_EVENT_ALL)) {
-    req.access = { user: req.user, permission: EventAccessType.Read };
+  const requestingUser = req.user?.from === 'sessionToken' ? req.user.account : null
+  if (requestingUser && !access.userHasPermission(requestingUser, MageEventPermission.READ_EVENT_ALL)) {
+    req.access = { userId: requestingUser.id, permission: EventAccessType.Read }
   }
-  next();
+  next()
 }
 
 /**
@@ -251,6 +253,7 @@ function EventRoutes(): express.Router {
         if (err) {
           return next(err);
         }
+        // TODO: scale: should avoid this for large user sets
         new api.Form(event).populateUserFields(function(err: any) {
           if (err) {
             return next(err);
diff --git a/service/src/models/event.d.ts b/service/src/models/event.d.ts
index 664c5f0de..de117218a 100644
--- a/service/src/models/event.d.ts
+++ b/service/src/models/event.d.ts
@@ -1,12 +1,13 @@
 import mongoose, { ToObjectOptions } from 'mongoose'
-import { UserDocument } from './user'
+import { UserDocument } from '../adapters/users/adapters.users.db.mongoose'
 import { MageEventId, MageEventAttrs, MageEventCreateAttrs, EventAccessType, EventRole } from '../entities/events/entities.events'
 import { Team, TeamMemberRole } from '../entities/teams/entities.teams'
 import { Form, FormField, FormFieldChoice } from '../entities/events/entities.events.forms'
 import { PageInfo } from '../utilities/paging';
+import { UserId } from '../entities/users/entities.users'
 
 export interface MageEventDocumentToObjectOptions extends ToObjectOptions {
-  access: { user: UserDocument, permission: EventAccessType }
+  access: { userId: UserId, permission: EventAccessType }
   projection: any
 }
 
@@ -67,7 +68,6 @@ export type Callback<Result = unknown> = (err: Error | null, result?: Result) =>
 export declare function count(options: TODO, callback: Callback<number>): void
 export declare function getEvents(options: TODO, callback: Callback<MageEventModelInstance[]>): void
 export declare function getById(id: MageEventId, options: TODO, callback: Callback<MageEventModelInstance | null>): void
-export declare function filterEventsByUserId(events: MageEventDocument[], userId: string, callback: Callback<MageEventModelInstance[]>): void
 export declare function create(event: MageEventCreateAttrs, user: Partial<UserDocument> & Pick<UserDocument, '_id'>, callback: Callback<MageEventModelInstance>): void
 export declare function addForm(eventId: MageEventId, form: any, callback: Callback<MageEventModelInstance>): void
 export declare function addLayer(event: MageEventDocument, layer: any, callback: Callback<MageEventModelInstance>): void
diff --git a/service/src/models/event.js b/service/src/models/event.js
index 44cf590ee..b1fcb1e87 100644
--- a/service/src/models/event.js
+++ b/service/src/models/event.js
@@ -208,11 +208,11 @@ function transform(event, ret, options) {
   // if read only permissions in event acl, only return users acl
   // TODO: move this business logic
   if (options.access) {
-    const roleOfUserOnEvent = ret.acl[options.access.user._id];
+    const roleOfUserOnEvent = ret.acl[options.access.userId];
     const rolesThatCanModify = rolesWithPermission('update').concat(rolesWithPermission('delete'));
     if (!roleOfUserOnEvent || rolesThatCanModify.indexOf(roleOfUserOnEvent) === -1) {
       const acl = {};
-      acl[options.access.user._id] = ret.acl[options.access.user._id];
+      acl[options.access.userId] = ret.acl[options.access.userId];
       ret.acl = acl;
     }
   }
@@ -268,18 +268,14 @@ function filterEventsByUserId(events, userId, callback) {
     if (err) return callback(err);
 
     const filteredEvents = events.filter(function (event) {
-      // Check if user has read access to the event based on
-      // being on a team that is in the event
+      // Check if user has read access to the event based on being on a team that is in the event
       if (event.teamIds.some(function (team) { return team.userIds.indexOf(userId) !== -1; })) {
         return true;
       }
-
-      // Check if user has read access to the event based on
-      // being in the events access control list
+      // Check if user has read access to the event based on being in the event access control list
       if (event.acl[userId] && rolesWithPermission('read').some(function (role) { return role === event.acl[userId]; })) {
         return true;
       }
-
       return false;
     });
 
@@ -297,9 +293,7 @@ exports.count = function (options, callback) {
   if (options.access) {
     const accesses = [];
     rolesWithPermission(options.access.permission).forEach(function (role) {
-      const access = {};
-      access['acl.' + options.access.user._id.toString()] = role;
-      accesses.push(access);
+      accesses.push({ [`acl.${options.access.userId}`]: role });
     });
     conditions['$or'] = accesses;
   }
@@ -335,9 +329,9 @@ exports.getEvents = function (options, callback) {
     const filters = [];
 
     // First filter out events user cannot access
-    if (options.access && options.access.user) {
+    if (options.access && options.access.userId) {
       filters.push(function (done) {
-        filterEventsByUserId(events, options.access.user._id, function (err, filteredEvents) {
+        filterEventsByUserId(events, options.access.userId, function (err, filteredEvents) {
           if (err) return done(err);
 
           events = filteredEvents;
@@ -411,9 +405,6 @@ exports.getById = function (id, options, callback) {
   });
 };
 
-// TODO probably should live in event api
-exports.filterEventsByUserId = filterEventsByUserId;
-
 function createObservationCollection(event) {
   log.info("Creating observation collection: " + event.collectionName + ' for event ' + event.name);
   mongoose.connection.db.createCollection(event.collectionName).then(() => {
@@ -549,22 +540,16 @@ exports.updateForm = function (event, form, callback) {
 exports.getMembers = async function (eventId, options) {
   const query = { _id: eventId };
   if (options.access) {
-    const accesses = [{
-      userIds: {
-        '$in': [options.access.user._id]
-      }
-    }];
-
+    const accesses = [
+      { userIds: { '$in': [ new mongoose.Types.ObjectId(options.access.userId) ] } }
+    ];
     rolesWithPermission(options.access.permission).forEach(role => {
-      const access = {};
-      access['acl.' + options.access.user._id.toString()] = role;
-      accesses.push(access);
+      accesses.push({ [`acl.${options.access.userId}`]: role });
     });
-
     query['$or'] = accesses;
   }
-  const event = await Event.findOne(query)
 
+  const event = await Event.findOne(query)
   if (event) {
     const { searchTerm } = options || {}
     const searchRegex = new RegExp(searchTerm, 'i')
@@ -609,22 +594,16 @@ exports.getMembers = async function (eventId, options) {
 exports.getNonMembers = async function (eventId, options) {
   const query = { _id: eventId };
   if (options.access) {
-    const accesses = [{
-      userIds: {
-        '$in': [options.access.user._id]
-      }
-    }];
-
+    const accesses = [
+      { userIds: { '$in': [ new mongoose.Types.ObjectId(options.access.userId) ] } }
+    ];
     rolesWithPermission(options.access.permission).forEach(role => {
-      const access = {};
-      access['acl.' + options.access.user._id.toString()] = role;
-      accesses.push(access);
+      accesses.push({ [`acl.${options.access.userId}`]: role });
     });
-
     query['$or'] = accesses;
   }
-  const event = await Event.findOne(query)
 
+  const event = await Event.findOne(query)
   if (event) {
     const { searchTerm } = options || {}
     const searchRegex = new RegExp(searchTerm, 'i')

From 7c17eefdab8532f17a450a87b7905a8ef4613d65 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 8 Nov 2024 22:36:39 -0700
Subject: [PATCH 146/183] refactor(service): users/auth: move passport idp web
 user typedef to protocol bindings module

---
 service/src/@types/express/index.d.ts         | 30 +++++++++++--------
 service/src/app.ts                            | 13 +++-----
 .../src/ingress/ingress.protocol.bindings.ts  | 29 ++++++++++++++++--
 service/src/ingress/ingress.protocol.local.ts |  2 +-
 service/src/ingress/ingress.protocol.oauth.ts |  7 +----
 5 files changed, 50 insertions(+), 31 deletions(-)

diff --git a/service/src/@types/express/index.d.ts b/service/src/@types/express/index.d.ts
index 41755dbe5..8c35753ce 100644
--- a/service/src/@types/express/index.d.ts
+++ b/service/src/@types/express/index.d.ts
@@ -1,20 +1,26 @@
-import { UserExpanded as MageUser } from '../../entities/users/entities.users'
-import { IdentityProviderUser } from '../../ingress/ingress.entities'
+import { UserExpanded } from '../../entities/users/entities.users'
+import { Session } from '../../ingress/ingress.entities'
 
 
-export type WebIngressUserFromSessionToken = { from: 'sessionToken', account: MageUser }
-export type WebIngressUserFromIdentityProvider = { from: 'identityProvider', account: IdentityProviderUser }
-/**
- * The `WebIngressUser` type determines the ingress path of the requesting user through the Passport middleware stack.
- * When the `mage` key is present, the requesting user authenticated using an established Mage session token.  When the
- * `identityProvider` key is present, the requesting user authenticated with a third party identity provider account
- * and will establish a new Mage session.
- */
-export type WebIngressUser = WebIngressUserFromSessionToken | WebIngressUserFromIdentityProvider
+export type AdmittedWebUser = {
+  account: UserExpanded
+  session: Session
+}
+
+declare global {
+  namespace Express {
+    interface User {
+      /**
+       * Mage populates the `admitted` user property when the user completes the ingress authentication flow through an
+       * identity provider and establishes a session.
+       */
+      admitted?: AdmittedWebUser
+    }
+  }
+}
 
 declare module 'express-serve-static-core' {
   export interface Request {
-    user?: Express.User & WebIngressUser
     token?: string
     // TODO: users-next: reconcile these two device properties and change to device entity
     provisionedDevice?: any
diff --git a/service/src/app.ts b/service/src/app.ts
index 42d988b29..b6d507054 100644
--- a/service/src/app.ts
+++ b/service/src/app.ts
@@ -38,7 +38,7 @@ import { MageEventRepositoryToken } from './plugins.api/plugins.api.events'
 import { FeedRepositoryToken, FeedServiceRepositoryToken, FeedServiceTypeRepositoryToken, FeedsAppServiceTokens } from './plugins.api/plugins.api.feeds'
 import { UserRepositoryToken } from './plugins.api/plugins.api.users'
 import { StaticIconRepositoryToken } from './plugins.api/plugins.api.icons'
-import { UserModel, MongooseUserRepository, UserModelName } from './adapters/users/adapters.users.db.mongoose'
+import { UserModel, MongooseUserRepository } from './adapters/users/adapters.users.db.mongoose'
 import { UserRepository, UserExpanded } from './entities/users/entities.users'
 import { EnvironmentService } from './entities/systemInfo/entities.systemInfo'
 import { WebRoutesHooks, GetAppRequestContext } from './plugins.api/plugins.api.web'
@@ -563,6 +563,7 @@ async function initWebLayer(repos: Repositories, app: AppLayer, webUIPlugins: st
       }
     }
   }
+  // TODO: users-next: initialize new ingress components
   const bearerAuth = webAuth.passport.authenticate('bearer')
 
   const settingsRoutes = SettingsRoutes(app.settings, appRequestFactory)
@@ -599,8 +600,7 @@ async function initWebLayer(repos: Repositories, app: AppLayer, webUIPlugins: st
     const context: observationsApi.ObservationRequestContext = {
       ...baseAppRequestContext(req),
       mageEvent: req[observationEventScopeKey]!.mageEvent,
-      // TODO: users-next
-      userId: req.user.id,
+      userId: req.user!.admitted!.account.id,
       deviceId: req.provisionedDeviceId,
       observationRepository: req[observationEventScopeKey]!.observationRepository
     }
@@ -640,12 +640,7 @@ async function initWebLayer(repos: Repositories, app: AppLayer, webUIPlugins: st
     return {
       requestToken: Symbol(),
       requestingPrincipal(): UserExpanded {
-        /*
-        TODO: users-next: this should ideally change so that the existing passport login
-        middleware applies the entity form of a user on the request rather than
-        the mongoose document instance
-        */
-        return { ...req.user?.account as UserExpanded }
+        return { ...req.user?.admitted?.account as UserExpanded }
       },
       locale(): Locale | null {
         return Object.freeze({
diff --git a/service/src/ingress/ingress.protocol.bindings.ts b/service/src/ingress/ingress.protocol.bindings.ts
index bc38ffd90..aa2bfe699 100644
--- a/service/src/ingress/ingress.protocol.bindings.ts
+++ b/service/src/ingress/ingress.protocol.bindings.ts
@@ -1,5 +1,27 @@
 import express from 'express'
-import { IdentityProvider } from './ingress.entities'
+import { IdentityProvider, IdentityProviderUser } from './ingress.entities'
+
+export type IdentityProviderAdmissionWebUser = {
+  idpName: string
+  account: IdentityProviderUser
+}
+
+declare global {
+  // eslint-disable-next-line @typescript-eslint/no-namespace
+  namespace Express {
+    interface User {
+      /**
+       * The ingress protocol must populate `req.user` with an object that has the `admittingFromIdentityProvider`
+       * property.  When using a Passport strategy middleware to handle the authentication flow, the protocol would
+       * invoke Passport's callback like
+       * ```
+       * done(null, { admittingFromIdentityProvider: { idpName: 'example', account: { ... } } })
+       * ```
+       */
+      admittingFromIdentityProvider?: IdentityProviderAdmissionWebUser
+    }
+  }
+}
 
 /**
  * `IngressProtocolWebBinding` is the binding of an authentication protocol's HTTP requests to an identity provider.
@@ -7,6 +29,7 @@ import { IdentityProvider } from './ingress.entities'
  * flow of HTTP messages between the Mage client, Mage server, and the identity provider's endpoints.
  */
 export interface IngressProtocolWebBinding {
-  readonly idp: IdentityProvider
+  idp: IdentityProvider
   handleRequest: express.RequestHandler
-}
\ No newline at end of file
+}
+
diff --git a/service/src/ingress/ingress.protocol.local.ts b/service/src/ingress/ingress.protocol.local.ts
index 551fcc617..f61da471e 100644
--- a/service/src/ingress/ingress.protocol.local.ts
+++ b/service/src/ingress/ingress.protocol.local.ts
@@ -20,7 +20,7 @@ function createAuthenticationMiddleware(localIdpAuthenticate: LocalIdpAuthentica
     if (authResult.success) {
       const localAccount = authResult.success
       const localIdpUser = userForLocalIdpAccount(localAccount)
-      return done(null, { from: 'identityProvider', account: localIdpUser })
+      return done(null, { admittingFromIdentityProvider: { idpName: 'local', account: localIdpUser } })
     }
     return done(authResult.error)
   }
diff --git a/service/src/ingress/ingress.protocol.oauth.ts b/service/src/ingress/ingress.protocol.oauth.ts
index c8c64d8d2..3869c2b50 100644
--- a/service/src/ingress/ingress.protocol.oauth.ts
+++ b/service/src/ingress/ingress.protocol.oauth.ts
@@ -3,7 +3,6 @@ import { InternalOAuthError, Strategy as OAuth2Strategy, StrategyOptions as OAut
 import base64 from 'base-64'
 import { IdentityProvider, IdentityProviderUser } from './ingress.entities'
 import { Authenticator } from 'passport'
-import { WebIngressUserFromIdentityProvider } from '../@types/express'
 
 export type OAuth2ProtocolSettings =
   Pick<OAuth2Options,
@@ -118,11 +117,7 @@ export function createWebBinding(idp: IdentityProvider, passport: Authenticator,
     const displayName = profile[profileKeys.displayName] || username
     const email = profile[profileKeys.email]
     const idpUser: IdentityProviderUser = { username, displayName, email, phones: [] }
-    const ingressUser: WebIngressUserFromIdentityProvider = {
-      from: 'identityProvider',
-      account: idpUser
-    }
-    return done(null, ingressUser)
+    return done(null, { admittingFromIdentityProvider: { idpName: idp.name, account: idpUser }})
   }
   const oauth2Strategy = new OAuth2ProfileStrategy(strategyOptions, profileURL, verify)
   return express.Router()

From 0216d1e6222416902e0989bdd2f218c38d898a53 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 8 Nov 2024 22:36:58 -0700
Subject: [PATCH 147/183] refactor(service): users/auth: rename session
 repository methods for consistency

---
 service/src/ingress/sessions.adapters.db.mongoose.ts | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/service/src/ingress/sessions.adapters.db.mongoose.ts b/service/src/ingress/sessions.adapters.db.mongoose.ts
index 69523e59e..541641153 100644
--- a/service/src/ingress/sessions.adapters.db.mongoose.ts
+++ b/service/src/ingress/sessions.adapters.db.mongoose.ts
@@ -37,7 +37,7 @@ export function SessionsMongooseRepository(conn: mongoose.Connection, collection
   const model = conn.model('Token', SessionSchema, collectionName)
   return Object.freeze({
     model,
-    async findSessionByToken(token: string): Promise<Session | null> {
+    async readSessionByToken(token: string): Promise<Session | null> {
       const doc = await model.findOne({ token }).lean()
       if (!doc) {
         return null
@@ -64,19 +64,19 @@ export function SessionsMongooseRepository(conn: mongoose.Connection, collection
       return await model.findOneAndUpdate(query, update,
         { upsert: true, new: true, populate: populateSessionUserRole })
     },
-    async removeSession(token: string): Promise<Session | null> {
-      const session = await this.findSessionByToken(token)
+    async deleteSession(token: string): Promise<Session | null> {
+      const session = await this.readSessionByToken(token)
       if (!session) {
         return null
       }
       const removed = await this.model.deleteOne({ token })
       return removed.deletedCount === 1 ? session : null
     },
-    async removeSessionsForUser(userId: UserId): Promise<number> {
+    async deleteSessionsForUser(userId: UserId): Promise<number> {
       const { deletedCount } = await model.deleteMany({ userId: new mongoose.Types.ObjectId(userId) })
       return deletedCount
     },
-    async removeSessionsForDevice(deviceId: string): Promise<number> {
+    async deleteSessionsForDevice(deviceId: string): Promise<number> {
       const { deletedCount } = await model.deleteMany({ deviceId: new mongoose.Types.ObjectId(deviceId) })
       return deletedCount
     }

From f6bc413aface452adde3b35e780d274e6cfec07d Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 8 Nov 2024 22:38:15 -0700
Subject: [PATCH 148/183] fix(service): db migration context log method was
 incorrectly typed

---
 service/src/@types/mongodb-migrations/index.d.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/@types/mongodb-migrations/index.d.ts b/service/src/@types/mongodb-migrations/index.d.ts
index a078246d9..060e96c84 100644
--- a/service/src/@types/mongodb-migrations/index.d.ts
+++ b/service/src/@types/mongodb-migrations/index.d.ts
@@ -76,7 +76,7 @@ declare module '@ngageoint/mongodb-migrations' {
 
   export type MigrationContext = {
     db: Db
-    log: Console
+    log: Console['log']
   }
 
   export interface Migration {

From 3c6afa484726322676f58c7c10261b85d45365d2 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 8 Nov 2024 22:38:52 -0700
Subject: [PATCH 149/183] refactor(service): users/auth: fix imports in local
 idp db adapter

---
 service/src/ingress/local-idp.adapters.db.mongoose.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/service/src/ingress/local-idp.adapters.db.mongoose.ts b/service/src/ingress/local-idp.adapters.db.mongoose.ts
index 8134ca8f3..db8359cfc 100644
--- a/service/src/ingress/local-idp.adapters.db.mongoose.ts
+++ b/service/src/ingress/local-idp.adapters.db.mongoose.ts
@@ -1,8 +1,8 @@
 "use strict";
 
 import mongoose from 'mongoose'
-import { IdentityProviderDocument, IdentityProviderModel } from './identity-providers.adapters.db.mongoose'
-import { LocalIdpDuplicateUsernameError, LocalIdpAccount, LocalIdpRepository, SecurityPolicy, localIdpSecurityPolicyFromIdenityProvider } from './local-idp.entities'
+import { IdentityProviderDocument, IdentityProviderModel } from './ingress.adapters.db.mongoose'
+import { LocalIdpDuplicateUsernameError, LocalIdpAccount, LocalIdpRepository, SecurityPolicy } from './local-idp.entities'
 
 const Schema = mongoose.Schema
 

From efd7d65d07380c4221533c9f7bab3bb014f5b267 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 8 Nov 2024 22:43:31 -0700
Subject: [PATCH 150/183] refactor(service): users/auth: add types for
 passport-openidconnect

---
 service/npm-shrinkwrap.json | 13 +++++++++++++
 service/package.json        |  1 +
 2 files changed, 14 insertions(+)

diff --git a/service/npm-shrinkwrap.json b/service/npm-shrinkwrap.json
index 3aad25a50..85cd33e8a 100644
--- a/service/npm-shrinkwrap.json
+++ b/service/npm-shrinkwrap.json
@@ -93,6 +93,7 @@
         "@types/passport-http-bearer": "^1.0.41",
         "@types/passport-local": "^1.0.38",
         "@types/passport-oauth2": "^1.4.17",
+        "@types/passport-openidconnect": "^0.1.3",
         "@types/sinon": "^9.0.4",
         "@types/sinon-chai": "^3.2.4",
         "@types/superagent": "^8.1.3",
@@ -2778,6 +2779,18 @@
         "@types/passport": "*"
       }
     },
+    "node_modules/@types/passport-openidconnect": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/@types/passport-openidconnect/-/passport-openidconnect-0.1.3.tgz",
+      "integrity": "sha512-k1Ni7bG/9OZNo2Qpjg2W6GajL+pww6ZPaNWMXfpteCX4dXf4QgaZLt2hjR5IiPrqwBT9+W8KjCTJ/uhGIoBx/g==",
+      "dev": true,
+      "dependencies": {
+        "@types/express": "*",
+        "@types/oauth": "*",
+        "@types/passport": "*",
+        "@types/passport-strategy": "*"
+      }
+    },
     "node_modules/@types/passport-strategy": {
       "version": "0.2.38",
       "resolved": "https://registry.npmjs.org/@types/passport-strategy/-/passport-strategy-0.2.38.tgz",
diff --git a/service/package.json b/service/package.json
index 0d030eaab..454509018 100644
--- a/service/package.json
+++ b/service/package.json
@@ -110,6 +110,7 @@
     "@types/passport-http-bearer": "^1.0.41",
     "@types/passport-local": "^1.0.38",
     "@types/passport-oauth2": "^1.4.17",
+    "@types/passport-openidconnect": "^0.1.3",
     "@types/sinon": "^9.0.4",
     "@types/sinon-chai": "^3.2.4",
     "@types/superagent": "^8.1.3",

From 52aa8631d769c3d8971867f3a50b79cf5415f9d0 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 14 Nov 2024 14:07:02 -0700
Subject: [PATCH 151/183] refactor(service): users/auth: handle state across
 external protocol flow requests

---
 .../ingress.adapters.controllers.web.ts       | 171 ++++++++++--------
 .../src/ingress/ingress.protocol.bindings.ts  |  27 ++-
 service/src/ingress/ingress.protocol.local.ts |  31 ++--
 service/src/ingress/ingress.protocol.oauth.ts |  40 ++--
 4 files changed, 161 insertions(+), 108 deletions(-)

diff --git a/service/src/ingress/ingress.adapters.controllers.web.ts b/service/src/ingress/ingress.adapters.controllers.web.ts
index 6869c53af..bcdf9ae37 100644
--- a/service/src/ingress/ingress.adapters.controllers.web.ts
+++ b/service/src/ingress/ingress.adapters.controllers.web.ts
@@ -4,23 +4,31 @@ import { Authenticator } from 'passport'
 import { Strategy as BearerStrategy } from 'passport-http-bearer'
 import { defaultHashUtil } from '../utilities/password-hashing'
 import { JWTService, Payload, TokenVerificationError, VerificationErrorReason, TokenAssertion } from './verification'
-import { invalidInput, InvalidInputError, MageError } from '../app.api/app.api.errors'
-import { IdentityProviderRepository } from './ingress.entities'
+import { invalidInput, InvalidInputError, MageError, permissionDenied } from '../app.api/app.api.errors'
+import { IdentityProvider, IdentityProviderRepository } from './ingress.entities'
 import { AdmitFromIdentityProviderOperation, EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
-import { IngressProtocolWebBinding } from './ingress.protocol.bindings'
+import { IngressProtocolWebBinding, IngressResponseType } from './ingress.protocol.bindings'
+import { createWebBinding as LocalBinding } from './ingress.protocol.local'
+import { createWebBinding as OAuthBinding } from './ingress.protocol.oauth'
 
 declare module 'express-serve-static-core' {
   interface Request {
     ingress?: IngressRequestContext
+    localEnrollment?: LocalEnrollmentContext
   }
 }
 
-type IngressRequestContext = { identityProviderService: IngressProtocolWebBinding } & (
-  | { state: 'init' }
-  | { state: 'localEnrollment', localEnrollment: LocalEnrollment }
-)
+enum UserAgentType {
+  MobileApp = 'MobileApp',
+  WebApp = 'WebApp'
+}
+
+type IngressRequestContext = {
+  idp: IdentityProvider
+  idpBinding: IngressProtocolWebBinding
+}
 
-type LocalEnrollment =
+type LocalEnrollmentContext =
   | {
     state: 'humanTokenVerified'
     captchaTokenPayload: Payload
@@ -30,7 +38,7 @@ type LocalEnrollment =
     subject: string
   }
 
-export type IngressOperations = {
+export type IngressUseCases = {
   enrollMyself: EnrollMyselfOperation
   admitFromIdentityProvider: AdmitFromIdentityProviderOperation
 }
@@ -40,78 +48,77 @@ export type IngressRoutes = {
   idpAdmission: express.Router
 }
 
-function bindingFor(idpName: string): IngressProtocolWebBinding {
-  throw new Error('unimplemented')
-}
-
-export function CreateIngressRoutes(ingressApp: IngressOperations, idpRepo: IdentityProviderRepository, tokenService: JWTService, passport: Authenticator): IngressRoutes {
+export function CreateIngressRoutes(ingressApp: IngressUseCases, idpRepo: IdentityProviderRepository, tokenService: JWTService, passport: Authenticator): IngressRoutes {
 
-  const captchaBearer = new BearerStrategy((token, done) => {
-    const expectation = {
-      subject: null,
-      expiration: null,
-      assertion: TokenAssertion.IsHuman
+  const idpBindings = new Map<string, IngressProtocolWebBinding>()
+  async function bindingFor(idpName: string): Promise<IngressProtocolWebBinding | null> {
+    const idp = await idpRepo.findIdpByName(idpName)
+    if (!idp) {
+      return null
     }
-    tokenService.verifyToken(token, expectation)
-      .then(payload => done(null, payload))
-      .catch(err => done(err))
-  })
-
-  // TODO: separate routers for /auth/idp/* and /api/users/signups/* for backward compatibility
+    if (idp.protocol === 'local') {
+      return LocalBinding(passport, )
+    }
+    throw new Error('unimplemented')
+  }
 
-  const routeToIdp = express.Router().all('/',
-    ((req, res, next) => {
-      const idpService = req.ingress?.identityProviderService
-      if (idpService) {
-        return idpService.handleRequest(req, res, next)
-      }
-      next(new Error(`no identity provider for ingress request: ${req.method} ${req.originalUrl}`))
-    }) as express.RequestHandler,
-    (async (err, req, res, next) => {
-      if (err) {
-        console.error('identity provider authentication error:', err)
-        return res.status(500).send('unexpected authentication result')
-      }
-      if (req.user?.from !== 'identityProvider') {
-        console.error('unexpected authentication user type:', req.user?.from)
-        return res.status(500).send('unexpected authentication result')
-      }
-      const identityProviderName = req.ingress!.identityProviderService!.idp.name
-      const identityProviderUser = req.user.account
-      const admission = await ingressApp.admitFromIdentityProvider({ identityProviderName, identityProviderUser })
-      if (admission.error) {
-        return next(admission.error)
-      }
-      const { admissionToken, mageAccount } = admission.success
-      /*
-      TODO: copied from redirecting protocols - cleanup and adapt here
-      local/ldap use direct json response
-      saml uses RelayState body property
-      oauth/oidc use state query parameter
-      can all use direct json response and handle redirect windows client side?
-      */
-      if (req.query.state === 'mobile') {
-        let uri;
-        if (!mageAccount.active || !mageAccount.enabled) {
-          uri = `mage://app/invalid_account?active=${mageAccount.active}&enabled=${mageAccount.enabled}`;
-        } else {
-          uri = `mage://app/authentication?token=${req.token}`
+  const routeToIdp = express.Router()
+    .all('/',
+      ((req, res, next) => {
+        const idpService = req.ingress?.idpBinding
+        if (!idpService) {
+          return next(new Error(`no identity provider for ingress request: ${req.method} ${req.originalUrl}`))
         }
-        res.redirect(uri);
-      } else {
-        res.render('authentication', { host: req.getRoot(), success: true, login: { token: req.token, user: req.user } });
-      }
-    }) as express.ErrorRequestHandler
-  )
+        if (req.path.endsWith('/signin')) {
+          const userAgentType: UserAgentType = req.params.state === 'mobile' ? UserAgentType.MobileApp : UserAgentType.WebApp
+          return idpService.beginIngressFlow(req, res, next, userAgentType)
+        }
+        idpService.handleIngressFlowRequest(req, res, next)
+      }) as express.RequestHandler,
+      (async (err, req, res, next) => {
+        if (err) {
+          console.error('identity provider authentication error:', err)
+          return res.status(500).send('unexpected authentication result')
+        }
+        if (!req.user?.admittingFromIdentityProvider) {
+          console.error('unexpected ingress user type:', req.user)
+          return res.status(500).send('unexpected authentication result')
+        }
+        const idpAdmission = req.user.admittingFromIdentityProvider
+        const { idpBinding, idp } = req.ingress!
+        const identityProviderUser = idpAdmission.account
+        const admission = await ingressApp.admitFromIdentityProvider({ identityProviderName: idp.name, identityProviderUser })
+        if (admission.error) {
+          return next(admission.error)
+        }
+        const { admissionToken, mageAccount } = admission.success
+        if (idpBinding.ingressResponseType === IngressResponseType.Direct) {
+          return res.json({ user: mageAccount, token: admissionToken })
+        }
+        if (idpAdmission.flowState === UserAgentType.MobileApp) {
+          if (mageAccount.active && mageAccount.enabled) {
+            return res.redirect(`mage://app/authentication?token=${req.token}`)
+          }
+          else {
+            return res.redirect(`mage://app/invalid_account?active=${mageAccount.active}&enabled=${mageAccount.enabled}`)
+          }
+        }
+        else if (idpAdmission.flowState === UserAgentType.WebApp) {
+          return res.render('authentication', { host: req.getRoot(), success: true, login: { token: admissionToken, user: mageAccount } })
+        }
+        return res.status(500).send('invalid authentication state')
+      }) as express.ErrorRequestHandler
+    )
 
   // TODO: mount to /auth
   const idpAdmission = express.Router()
   idpAdmission.use('/:identityProviderName',
-    (req, res, next) => {
+    async (req, res, next) => {
       const idpName = req.params.identityProviderName
-      const idpService = bindingFor(idpName)
-      if (idpService) {
-        req.ingress = { state: 'init', identityProviderService: idpService }
+      const idp = await idpRepo.findIdpByName(idpName)
+      const idpBinding = await bindingFor(idpName)
+      if (idpBinding && idp) {
+        req.ingress = { idpBinding, idp }
         return next()
       }
       res.status(404).send(`${idpName} not found`)
@@ -149,6 +156,17 @@ export function CreateIngressRoutes(ingressApp: IngressOperations, idpRepo: Iden
       }
     })
 
+  const captchaBearer = new BearerStrategy((token, done) => {
+    const expectation = {
+      subject: null,
+      expiration: null,
+      assertion: TokenAssertion.IsHuman
+    }
+    tokenService.verifyToken(token, expectation)
+      .then(payload => done(null, payload))
+      .catch(err => done(err))
+  })
+
   // TODO: mount to /api/users/signups/verifications
   localEnrollment.route('/signups/verifications')
     .post(
@@ -163,19 +181,16 @@ export function CreateIngressRoutes(ingressApp: IngressOperations, idpRepo: Iden
           if (!captchaTokenPayload) {
             return res.status(400).send('Missing captcha token')
           }
-          req.ingress = {
-            ...req.ingress!,
-            state: 'localEnrollment',
-            localEnrollment: { state: 'humanTokenVerified', captchaTokenPayload } }
+          req.localEnrollment = { state: 'humanTokenVerified', captchaTokenPayload }
           next()
         })(req, res, next)
       },
       async (req, res, next) => {
         try {
-          if (req.ingress?.state !== 'localEnrollment' || req.ingress.localEnrollment.state !== 'humanTokenVerified') {
+          if (req.localEnrollment?.state !== 'humanTokenVerified') {
             return res.status(500).send('invalid ingress state')
           }
-          const tokenPayload = req.ingress.localEnrollment.captchaTokenPayload
+          const tokenPayload = req.localEnrollment.captchaTokenPayload
           const hashedCaptchaText = tokenPayload.captcha as string
           const userCaptchaText = req.body.captchaText
           const isHuman = await defaultHashUtil.validPassword(userCaptchaText, hashedCaptchaText)
diff --git a/service/src/ingress/ingress.protocol.bindings.ts b/service/src/ingress/ingress.protocol.bindings.ts
index aa2bfe699..52dcf875f 100644
--- a/service/src/ingress/ingress.protocol.bindings.ts
+++ b/service/src/ingress/ingress.protocol.bindings.ts
@@ -1,9 +1,10 @@
 import express from 'express'
-import { IdentityProvider, IdentityProviderUser } from './ingress.entities'
+import { IdentityProviderUser } from './ingress.entities'
 
 export type IdentityProviderAdmissionWebUser = {
   idpName: string
-  account: IdentityProviderUser
+  account: IdentityProviderUser | undefined
+  flowState?: string | undefined
 }
 
 declare global {
@@ -23,13 +24,31 @@ declare global {
   }
 }
 
+export enum IngressResponseType {
+  Direct = 'Direct',
+  Redirect = 'Redirect'
+}
+
 /**
  * `IngressProtocolWebBinding` is the binding of an authentication protocol's HTTP requests to an identity provider.
  * The protocol uses the identity provider settings to determine the identity provider's endpoints and orchestrate the
  * flow of HTTP messages between the Mage client, Mage server, and the identity provider's endpoints.
  */
 export interface IngressProtocolWebBinding {
-  idp: IdentityProvider
-  handleRequest: express.RequestHandler
+  ingressResponseType: IngressResponseType
+  /**
+   * This function initiates the protocol's ingress process, which starts with a request to the `/signin` path of the
+   * IDP's context, e.g., `GET /auth/google-oidc/signin`.
+   *
+   * The `flowState` parameter is a URL-safe, percent-encoded string value which holds any state information the app
+   * needs to persist across multiple ingress protocol requests.
+   * This is primarily for saving information about how Mage delivers the final ingress result to the client, such as
+   * a direct response, or a redirect URL suitable for the modile or web apps.
+   * Different protocols have different ways of persisting state across requests, such as the OAuth/OpenID Connect
+   * `state` parameter and the SAML `RelayState` body attribute.  The protocol must store this value and return the
+   * value in the {@link IdentityProviderAdmissionWebUser#flowState admission result}.
+   */
+  beginIngressFlow(req: express.Request, res: express.Response, next: express.NextFunction, flowState: string | undefined): any
+  handleIngressFlowRequest: express.RequestHandler
 }
 
diff --git a/service/src/ingress/ingress.protocol.local.ts b/service/src/ingress/ingress.protocol.local.ts
index f61da471e..05f6ab5b5 100644
--- a/service/src/ingress/ingress.protocol.local.ts
+++ b/service/src/ingress/ingress.protocol.local.ts
@@ -4,6 +4,7 @@ import { Strategy as LocalStrategy, VerifyFunction as LocalStrategyVerifyFunctio
 import { LocalIdpAccount } from './local-idp.entities'
 import { IdentityProviderUser } from './ingress.entities'
 import { LocalIdpAuthenticateOperation } from './local-idp.app.api'
+import { IngressProtocolWebBinding, IngressResponseType } from './ingress.protocol.bindings'
 
 
 function userForLocalIdpAccount(account: LocalIdpAccount): IdentityProviderUser {
@@ -14,13 +15,13 @@ function userForLocalIdpAccount(account: LocalIdpAccount): IdentityProviderUser
   }
 }
 
-function createAuthenticationMiddleware(localIdpAuthenticate: LocalIdpAuthenticateOperation): passport.Strategy {
+function createLocalStrategy(localIdpAuthenticate: LocalIdpAuthenticateOperation, flowState: string | undefined): passport.Strategy {
   const verify: LocalStrategyVerifyFunction = async function LocalIngressProtocolVerify(username, password, done) {
     const authResult = await localIdpAuthenticate({ username, password })
     if (authResult.success) {
       const localAccount = authResult.success
       const localIdpUser = userForLocalIdpAccount(localAccount)
-      return done(null, { admittingFromIdentityProvider: { idpName: 'local', account: localIdpUser } })
+      return done(null, { admittingFromIdentityProvider: { idpName: 'local', account: localIdpUser, flowState } })
     }
     return done(authResult.error)
   }
@@ -39,13 +40,21 @@ const validateSigninRequest: express.RequestHandler = function LocalProtocolIngr
   next()
 }
 
-export function createWebBinding(passport: passport.Authenticator, localIdpAuthenticate: LocalIdpAuthenticateOperation): express.RequestHandler {
-  const authStrategy = createAuthenticationMiddleware(localIdpAuthenticate)
-  const handleRequest = express.Router()
-    .post('/signin',
-      express.urlencoded(),
-      validateSigninRequest,
-      passport.authenticate(authStrategy)
-    )
-  return handleRequest
+export function createWebBinding(passport: passport.Authenticator, localIdpAuthenticate: LocalIdpAuthenticateOperation): IngressProtocolWebBinding {
+  return {
+    ingressResponseType: IngressResponseType.Direct,
+    beginIngressFlow: (req, res, next, flowState): any => {
+      const authStrategy = createLocalStrategy(localIdpAuthenticate, flowState)
+      const applyLocalProtocol = express.Router()
+        .post('/*',
+          express.urlencoded(),
+          validateSigninRequest,
+          passport.authenticate(authStrategy)
+        )
+      applyLocalProtocol(req, res, next)
+    },
+    handleIngressFlowRequest(req, res): any {
+      return res.status(400).send('invalid local ingress request')
+    }
+  }
 }
diff --git a/service/src/ingress/ingress.protocol.oauth.ts b/service/src/ingress/ingress.protocol.oauth.ts
index 3869c2b50..9946c2271 100644
--- a/service/src/ingress/ingress.protocol.oauth.ts
+++ b/service/src/ingress/ingress.protocol.oauth.ts
@@ -3,6 +3,7 @@ import { InternalOAuthError, Strategy as OAuth2Strategy, StrategyOptions as OAut
 import base64 from 'base-64'
 import { IdentityProvider, IdentityProviderUser } from './ingress.entities'
 import { Authenticator } from 'passport'
+import { IngressProtocolWebBinding, IngressResponseType } from './ingress.protocol.bindings'
 
 export type OAuth2ProtocolSettings =
   Pick<OAuth2Options,
@@ -59,7 +60,7 @@ class OAuth2ProfileStrategy extends OAuth2Strategy {
         done(null, profile)
       }
       catch (err) {
-        log.error('error parsing oauth profile', err)
+        console.error('error parsing oauth profile', err)
         done(err)
       }
     })
@@ -81,11 +82,13 @@ function applyDefaultProtocolSettings(idp: IdentityProvider): OAuth2ProtocolSett
   return settings
 }
 
+type OAuth2Info = { state: string }
+
 /**
  * The `baseUrl` parameter is the URL at which Mage will mount the returned `express.Router`, including any
  * distinguishing component of the given `IdentityProvider`, without a trailing slash, e.g. `/auth/example-idp`.
  */
-export function createWebBinding(idp: IdentityProvider, passport: Authenticator, baseUrl: string): express.Router {
+export function createWebBinding(idp: IdentityProvider, passport: Authenticator, baseUrl: string): IngressProtocolWebBinding {
   const settings = applyDefaultProtocolSettings(idp)
   const profileURL = settings.profileURL
   const customHeaders = settings.headers?.basic ? {
@@ -110,24 +113,31 @@ export function createWebBinding(idp: IdentityProvider, passport: Authenticator,
     const profile = profileResponse.json
     const profileKeys = settings.profile
     if (!profile[profileKeys.id]) {
-      log.warn("JSON: " + JSON.stringify(profile) + " RAW: " + profileResponse.raw);
-      return done(`OAuth2 user profile does not contain id property named ${profileKeys.id}`)
+      console.error(`oauth2 profile missing id for key ${profileKeys.id}:\n`, JSON.stringify(profile, null, 2))
+      return done(new Error(`oauth2 user profile does not contain id property ${profileKeys.id}`))
     }
     const username = profile[profileKeys.id]
     const displayName = profile[profileKeys.displayName] || username
     const email = profile[profileKeys.email]
     const idpUser: IdentityProviderUser = { username, displayName, email, phones: [] }
-    return done(null, { admittingFromIdentityProvider: { idpName: idp.name, account: idpUser }})
+    return done(null, { admittingFromIdentityProvider: { idpName: idp.name, account: idpUser, flowState: '' }})
   }
   const oauth2Strategy = new OAuth2ProfileStrategy(strategyOptions, profileURL, verify)
-  return express.Router()
-    .get('/signin',
-      /*
-      TODO:
-      this used to pass state in the options argument like { state: req.query.state }
-      to propagate the mobile clients passing state=mobile. test whether this is actually necessary
-      */
-      passport.authenticate(oauth2Strategy, { scope: settings.scope })
-    )
-    .get('/callback', passport.authenticate(oauth2Strategy))
+  const handleIngressFlowRequest = express.Router()
+    .get('/callback', (req, res, next) => {
+      passport.authenticate(oauth2Strategy,
+        (err: Error | null | undefined, account: IdentityProviderUser, info: OAuth2Info) => {
+          if (err) {
+            return next(err)
+          }
+          req.user = { admittingFromIdentityProvider: { idpName: idp.name, account, flowState: info.state }}
+        })(req, res, next)
+    })
+  return {
+    ingressResponseType: IngressResponseType.Redirect,
+    beginIngressFlow(req, res, next, flowState): any {
+      passport.authenticate(oauth2Strategy, { state: flowState })(req, res, next)
+    },
+    handleIngressFlowRequest
+  }
 }
\ No newline at end of file

From 1794201dc72d12b508dbd3029e31cc2c37fa5825 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 14 Nov 2024 14:45:36 -0700
Subject: [PATCH 152/183] refactor(service): users/auth: migrate oidc ingress
 protocol to trypescript and new auth scheme

---
 service/src/ingress/ingress.protocol.oidc.ts | 264 ++++++++-----------
 1 file changed, 109 insertions(+), 155 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.oidc.ts b/service/src/ingress/ingress.protocol.oidc.ts
index 979026786..50054a82b 100644
--- a/service/src/ingress/ingress.protocol.oidc.ts
+++ b/service/src/ingress/ingress.protocol.oidc.ts
@@ -1,168 +1,122 @@
-const OpenIdConnectStrategy = require('passport-openidconnect').Strategy
-  , log = require('winston')
-  , User = require('../models/user')
-  , Role = require('../models/role')
-  , TokenAssertion = require('./verification').TokenAssertion
-  , api = require('../api')
-  , { app, passport, tokenService } = require('./index');
-
-function configure(strategy) {
-  log.info(`Configuring ${strategy.title} authentication`);
-
-  passport.use(strategy.name, new OpenIdConnectStrategy({
-    clientID: strategy.settings.clientID,
-    clientSecret: strategy.settings.clientSecret,
-    issuer: strategy.settings.issuer,
-    authorizationURL: strategy.settings.authorizationURL,
-    tokenURL: strategy.settings.tokenURL,
-    userInfoURL: strategy.settings.profileURL,
-    callbackURL: `/auth/${strategy.name}/callback`,
-    scope: strategy.settings.scope
-  }, function (issuer, uiProfile, profile, context, idToken, accessToken, refreshToken, params, done) {
-    const jsonProfile = uiProfile._json
-    const profileId = jsonProfile[strategy.settings.profile.id];
-    if (!profileId) {
-      log.warn(JSON.stringify(jsonProfile));
-      return done(`OIDC user profile does not contain id property ${strategy.settings.profile.id}`);
+import express from 'express'
+import passport from 'passport'
+import OpenIdConnectStrategy from 'passport-openidconnect'
+import { IdentityProvider, IdentityProviderUser } from './ingress.entities'
+import { IdentityProviderAdmissionWebUser, IngressProtocolWebBinding, IngressResponseType } from './ingress.protocol.bindings'
+
+
+export type OpenIdConnectProtocolSettings =
+  Pick<
+    OpenIdConnectStrategy.StrategyOptions,
+    'clientID' | 'clientSecret' | 'issuer' | 'authorizationURL' | 'tokenURL' | 'scope'
+  > &
+  {
+    profileURL: string,
+    profile: {
+      displayName?: string
+      email?: string
+      id?: string
     }
-
-    // TODO: users-next
-    User.getUserByAuthenticationStrategy(strategy.type, profileId, function (err, user) {
-      if (err) return done(err);
-
-      if (!user) {
-        // Create an account for the user
-        Role.getRole('USER_ROLE', function (err, role) {
-          if (err) return done(err);
-
-          const user = {
-            username: profileId,
-            displayName: jsonProfile[strategy.settings.profile.displayName] || profileId,
-            email: jsonProfile[strategy.settings.profile.email],
-            active: false,
-            roleId: role._id,
-            authentication: {
-              type: strategy.name,
-              id: profileId,
-              authenticationConfiguration: {
-                name: strategy.name
-              }
-            }
-          };
-          // TODO: users-next
-          new api.User().create(user).then(newUser => {
-            if (!newUser.authentication.authenticationConfiguration.enabled) {
-              log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
-              return done(null, false, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-            }
-            return done(null, newUser);
-          }).catch(err => done(err));
-        });
-      } else if (!user.active) {
-        return done(null, user, { message: "User is not approved, please contact your MAGE administrator to approve your account." });
-      } else if (!user.authentication.authenticationConfiguration.enabled) {
-        log.warn(user.authentication.authenticationConfiguration.title + " authentication is not enabled");
-        return done(null, user, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-      } else {
-        return done(null, user);
-      }
-    });
-  }));
-
-  function authenticate(req, res, next) {
-    passport.authenticate(strategy.name, function (err, user, info = {}) {
-      if (err) return next(err);
-
-      // TODO, this is a workaround for openidconnect library killing the app state
-      req.query.state = info.state
-
-      req.user = user;
-
-      // For inactive or disabled accounts don't generate an authorization token
-      if (!user.active || !user.enabled) {
-        log.warn('Failed user login attempt: User ' + user.username + ' account is inactive or disabled.');
-        return next();
-      }
-
-      if (!user.authentication.authenticationConfigurationId) {
-        log.warn('Failed user login attempt: ' + user.authentication.type + ' is not configured');
-        return next();
-      }
-
-      if (!user.authentication.authenticationConfiguration.enabled) {
-        log.warn('Failed user login attempt: Authentication ' + user.authentication.authenticationConfiguration.title + ' is disabled.');
-        return next();
-      }
-
-      tokenService.generateToken(user._id.toString(), TokenAssertion.Authorized, 60 * 5)
-        .then(token => {
-          req.token = token;
-          req.user = user;
-          req.info = info
-          next();
-        }).catch(err => {
-          next(err);
-        });
-    })(req, res, next);
   }
 
-  app.get(`/auth/${strategy.name}/callback`,
-    authenticate,
-    function (req, res) {
-      if (req.query.state === 'mobile') {
-        let uri;
-        if (!req.user.active || !req.user.enabled) {
-          uri = `mage://app/invalid_account?active=${req.user.active}&enabled=${req.user.enabled}`;
-        } else {
-          uri = `mage://app/authentication?token=${req.token}`
-        }
-
-        res.redirect(uri);
-      } else {
-        res.render('authentication', { host: req.getRoot(), success: true, login: { token: req.token, user: req.user } });
-      }
-    }
-  );
+function copyProtocolSettings(from: OpenIdConnectProtocolSettings): OpenIdConnectProtocolSettings {
+  const copy = { ...from }
+  copy.profile = { ...from.profile }
+  if (Array.isArray(from.scope)) {
+    copy.scope = [ ...from.scope ]
+  }
+  return copy
 }
 
-function setDefaults(strategy) {
-  //openid must be included in scope
-  if (!strategy.settings.scope) {
-    strategy.settings.scope = ['openid'];
-  } else {
-    if (!strategy.settings.scope.includes('openid')) {
-      strategy.settings.scope.push('openid');
-    }
+function applyDefaultProtocolSettings(idp: IdentityProvider): OpenIdConnectProtocolSettings {
+  const settings = copyProtocolSettings(idp.protocolSettings as OpenIdConnectProtocolSettings)
+  if (!settings.scope) {
+    settings.scope = [ 'openid' ]
   }
-
-  if (!strategy.settings.profile) {
-    strategy.settings.profile = {};
+  else if (Array.isArray(settings.scope) && !settings.scope.includes('openid')) {
+    settings.scope = [ ...settings.scope, 'openid' ]
+  }
+  else if (typeof settings.scope === 'string' && settings.scope !== 'openid') {
+    settings.scope = [ settings.scope, 'openid' ]
   }
-  if (!strategy.settings.profile.displayName) {
-    strategy.settings.profile.displayName = 'name';
+  const profile = settings.profile
+  if (!profile.displayName) {
+    profile.displayName = 'displayName'
   }
-  if (!strategy.settings.profile.email) {
-    strategy.settings.profile.email = 'email';
+  if (!profile.email) {
+    profile.email = 'email'
   }
-  if (!strategy.settings.profile.id) {
-    strategy.settings.profile.id = 'sub';
+  if (!profile.id) {
+    profile.id = 'sub';
   }
+  return settings
 }
 
-function initialize(strategy) {
-  configure(strategy);
-  setDefaults(strategy);
-
-  app.get(`/auth/${strategy.name}/signin`,
-    function (req, res, next) {
-      passport.authenticate(strategy.name, {
-        scope: strategy.settings.scope,
-        state: req.query.state
-      })(req, res, next);
+export function createWebBinding(idp: IdentityProvider, passport: passport.Authenticator, baseUrl: string): IngressProtocolWebBinding {
+  const settings = applyDefaultProtocolSettings(idp)
+  const verify: OpenIdConnectStrategy.VerifyFunction = (
+    issuer: string,
+    uiProfile: any,
+    idProfile: object,
+    context: object,
+    idToken: string | object,
+    accessToken: string | object,
+    refreshToken: string,
+    params: any,
+    done: OpenIdConnectStrategy.VerifyCallback
+  ) => {
+    const jsonProfile = uiProfile._json
+    const idpAccountId = jsonProfile[settings.profile.id!]
+    if (!idpAccountId) {
+      const message = `user profile from oidc identity provider ${idp.name} does not contain id property ${settings.profile.id}`
+      console.error(message, JSON.stringify(jsonProfile, null, 2))
+      return done(new Error(message))
     }
-  );
-};
-
-module.exports = {
-  initialize
-}
\ No newline at end of file
+    const idpUser: IdentityProviderUser = {
+      username: idpAccountId,
+      displayName: jsonProfile[settings.profile.displayName!] || idpAccountId,
+      email: jsonProfile[settings.profile.email!],
+      phones: [],
+      idpAccountId
+    }
+    done(null, { admittingFromIdentityProvider: { idpName: idp.name, account: idpUser } } )
+  }
+  const oidcStrategy = new OpenIdConnectStrategy(
+    {
+      clientID: settings.clientID,
+      clientSecret: settings.clientSecret,
+      issuer: settings.issuer,
+      authorizationURL: settings.authorizationURL,
+      tokenURL: settings.tokenURL,
+      userInfoURL: settings.profileURL,
+      callbackURL: `${baseUrl}/callback`,
+      scope: settings.scope
+    },
+    verify
+  )
+  const handleIngressFlowRequest = express.Router()
+    .get('/callback', (req, res, next) => {
+      const finishIngressFlow = passport.authenticate(
+        oidcStrategy,
+        (err: Error | null, user: IdentityProviderAdmissionWebUser, info: { state: string | undefined }) => {
+          if (err) {
+            return next(err)
+          }
+          const idpUserWithState: IdentityProviderAdmissionWebUser = {
+            ...user,
+            flowState: info.state
+          }
+          req.user = { admittingFromIdentityProvider: idpUserWithState }
+          next()
+        }
+      )
+      finishIngressFlow(req, res, next)
+    })
+  return {
+    ingressResponseType: IngressResponseType.Redirect,
+    beginIngressFlow(req, res, next, flowState): any {
+      passport.authenticate(oidcStrategy, { state: flowState })(req, res, next)
+    },
+    handleIngressFlowRequest
+  }
+}

From 3acf16b5a47903a66416b6f1bbc86cd2ab211ecc Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 14 Nov 2024 14:46:09 -0700
Subject: [PATCH 153/183] refactor(service): users/auth: call next middleware
 in oauth ingress protocol

---
 service/src/ingress/ingress.protocol.oauth.ts | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.oauth.ts b/service/src/ingress/ingress.protocol.oauth.ts
index 9946c2271..6d0583b4b 100644
--- a/service/src/ingress/ingress.protocol.oauth.ts
+++ b/service/src/ingress/ingress.protocol.oauth.ts
@@ -125,13 +125,17 @@ export function createWebBinding(idp: IdentityProvider, passport: Authenticator,
   const oauth2Strategy = new OAuth2ProfileStrategy(strategyOptions, profileURL, verify)
   const handleIngressFlowRequest = express.Router()
     .get('/callback', (req, res, next) => {
-      passport.authenticate(oauth2Strategy,
+      const finishIngressFlow = passport.authenticate(
+        oauth2Strategy,
         (err: Error | null | undefined, account: IdentityProviderUser, info: OAuth2Info) => {
           if (err) {
             return next(err)
           }
           req.user = { admittingFromIdentityProvider: { idpName: idp.name, account, flowState: info.state }}
-        })(req, res, next)
+          next()
+        }
+      )
+      finishIngressFlow(req, res, next)
     })
   return {
     ingressResponseType: IngressResponseType.Redirect,

From 1f82c17f5ac970da3b91c9549aede82b4e752f4f Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 14 Nov 2024 23:30:55 -0700
Subject: [PATCH 154/183] refactor(service): users/auth: migrate saml ingress
 protocol to trypescript and new auth scheme

---
 service/src/ingress/ingress.protocol.saml.ts | 340 ++++++-------------
 1 file changed, 106 insertions(+), 234 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.saml.ts b/service/src/ingress/ingress.protocol.saml.ts
index 2376bd1f8..521703152 100644
--- a/service/src/ingress/ingress.protocol.saml.ts
+++ b/service/src/ingress/ingress.protocol.saml.ts
@@ -1,251 +1,123 @@
-const SamlStrategy = require('@node-saml/passport-saml').Strategy
-  , log = require('winston')
-  , User = require('../models/user')
-  , Role = require('../models/role')
-  , TokenAssertion = require('./verification').TokenAssertion
-  , api = require('../api')
-  , AuthenticationInitializer = require('./index')
+import express from 'express'
+import { Authenticator } from 'passport'
+import { SamlConfig, Strategy as SamlStrategy, VerifyWithRequest } from '@node-saml/passport-saml'
+import { IdentityProvider, IdentityProviderUser } from './ingress.entities'
+import { IngressProtocolWebBinding, IngressResponseType } from './ingress.protocol.bindings'
 
-function configure(strategy) {
-  log.info('Configuring ' + strategy.title + ' authentication');
 
-  const options = {
-    path: `/auth/${strategy.name}/callback`,
-    entryPoint: strategy.settings.entryPoint,
-    cert: strategy.settings.cert,
-    issuer: strategy.settings.issuer
-  }
-  if (strategy.settings.privateKey) {
-    options.privateKey = strategy.settings.privateKey;
-  }
-  if (strategy.settings.decryptionPvk) {
-    options.decryptionPvk = strategy.settings.decryptionPvk;
-  }
-  if (strategy.settings.signatureAlgorithm) {
-    options.signatureAlgorithm = strategy.settings.signatureAlgorithm;
-  }
-  if(strategy.settings.audience) {
-    options.audience = strategy.settings.audience;
-  }
-  if(strategy.settings.identifierFormat) {
-    options.identifierFormat = strategy.settings.identifierFormat;
-  }
-  if(strategy.settings.acceptedClockSkewMs) {
-    options.acceptedClockSkewMs = strategy.settings.acceptedClockSkewMs;
-  }
-  if(strategy.settings.attributeConsumingServiceIndex) {
-    options.attributeConsumingServiceIndex = strategy.settings.attributeConsumingServiceIndex;
-  }
-  if(strategy.settings.disableRequestedAuthnContext) {
-    options.disableRequestedAuthnContext = strategy.settings.disableRequestedAuthnContext;
-  }
-  if(strategy.settings.authnContext) {
-    options.authnContext = strategy.settings.authnContext;
-  }
-  if(strategy.settings.forceAuthn) {
-    options.forceAuthn = strategy.settings.forceAuthn;
-  }
-  if(strategy.settings.skipRequestCompression) {
-    options.skipRequestCompression = strategy.settings.skipRequestCompression;
-  }
-  if(strategy.settings.authnRequestBinding) {
-    options.authnRequestBinding = strategy.settings.authnRequestBinding;
-  }
-  if(strategy.settings.RACComparison) {
-    options.RACComparison = strategy.settings.RACComparison;
-  }
-  if(strategy.settings.providerName) {
-    options.providerName = strategy.settings.providerName;
-  }
-  if(strategy.settings.idpIssuer) {
-    options.idpIssuer = strategy.settings.idpIssuer;
+type SamlProfileKeys = {
+  id?: string
+  email?: string
+  displayName?: string
+}
+
+type SamlProtocolSettings =
+  Pick<
+    SamlConfig,
+    | 'path'
+    | 'entryPoint'
+    | 'cert'
+    | 'issuer'
+    | 'privateKey'
+    | 'decryptionPvk'
+    | 'signatureAlgorithm'
+    | 'audience'
+    | 'identifierFormat'
+    | 'acceptedClockSkewMs'
+    | 'attributeConsumingServiceIndex'
+    | 'disableRequestedAuthnContext'
+    | 'authnContext'
+    | 'forceAuthn'
+    | 'skipRequestCompression'
+    | 'authnRequestBinding'
+    | 'racComparison'
+    | 'providerName'
+    | 'idpIssuer'
+    | 'validateInResponseTo'
+    | 'requestIdExpirationPeriodMs'
+    | 'logoutUrl'
+  >
+  & {
+    profile: SamlProfileKeys
+  }
+
+function copyProtocolSettings(from: SamlProtocolSettings): SamlProtocolSettings {
+  const copy = { ...from }
+  copy.profile = { ...from.profile }
+  return copy
+}
+
+function applyDefaultProtocolSettings(idp: IdentityProvider): SamlProtocolSettings {
+  const settings = copyProtocolSettings(idp.protocolSettings as SamlProtocolSettings)
+  if (!settings.profile) {
+    settings.profile = {}
   }
-  if(strategy.settings.validateInResponseTo) {
-    options.validateInResponseTo = strategy.settings.validateInResponseTo;
+  if (!settings.profile.displayName) {
+    settings.profile.displayName = 'email'
   }
-  if(strategy.settings.requestIdExpirationPeriodMs) {
-    options.requestIdExpirationPeriodMs = strategy.settings.requestIdExpirationPeriodMs;
+  if (!settings.profile.email) {
+    settings.profile.email = 'email'
   }
-  if(strategy.settings.logoutUrl) {
-    options.logoutUrl = strategy.settings.logoutUrl;
+  if (!settings.profile.id) {
+    settings.profile.id = 'uid'
   }
+  return settings
+}
 
-  AuthenticationInitializer.passport.use(new SamlStrategy(options, function (profile, done) {
-    const uid = profile[strategy.settings.profile.id];
-
-    if (!uid) {
-      log.warn('Failed to find property uid. SAML profile keys ' + Object.keys(profile));
-      return done('Failed to load user id from SAML profile');
-    }
-
-    // TODO: users-next
-    User.getUserByAuthenticationStrategy(strategy.type, uid, function (err, user) {
-      if (err) return done(err);
-
-      if (!user) {
-        // Create an account for the user
-        Role.getRole('USER_ROLE', function (err, role) {
-          if (err) return done(err);
-
-          const user = {
-            username: uid,
-            displayName: profile[strategy.settings.profile.displayName],
-            email: profile[strategy.settings.profile.email],
-            active: false,
-            roleId: role._id,
-            authentication: {
-              type: strategy.name,
-              id: uid,
-              authenticationConfiguration: {
-                name: strategy.name
-              }
-            }
-          };
-          // TODO: users-next
-          new api.User().create(user).then(newUser => {
-            if (!newUser.authentication.authenticationConfiguration.enabled) {
-              log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
-              return done(null, false, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-            }
-            return done(null, newUser);
-          }).catch(err => done(err));
-        });
-      } else if (!user.active) {
-        return done(null, user, { message: "User is not approved, please contact your MAGE administrator to approve your account." });
-      } else if (!user.authentication.authenticationConfiguration.enabled) {
-        log.warn(user.authentication.authenticationConfiguration.title + " authentication is not enabled");
-        return done(null, user, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-      } else {
-        return done(null, user);
-      }
-    });
-  }));
-
-  function authenticate(req, res, next) {
-    AuthenticationInitializer.passport.authenticate(strategy.name, function (err, user, info = {}) {
-      if (err) {
-        console.error('saml: authentication error', err);
-        return next(err);
-      }
-
-      req.user = user;
-
-      // For inactive or disabled accounts don't generate an authorization token
-      if (!user.active || !user.enabled) {
-        log.warn('Failed user login attempt: User ' + user.username + ' account is inactive or disabled.');
-        return next();
+export function createWebBinding(idp: IdentityProvider, passport: Authenticator, baseUrlPath: string): IngressProtocolWebBinding {
+  const { profile: profileKeys, ...settings } = applyDefaultProtocolSettings(idp)
+  // TODO: this will need the the saml callback override change
+  settings.path = `${baseUrlPath}/callback`
+  const samlStrategy = new SamlStrategy(settings,
+    (function samlSignIn(req, profile, done) {
+      if (!profile) {
+        return done(new Error('missing saml profile'))
       }
-
-      if (!user.authentication.authenticationConfigurationId) {
-        log.warn('Failed user login attempt: ' + user.authentication.type + ' is not configured');
-        return next();
+      const uid = profile[profileKeys.id!]
+      if (!uid || typeof uid !== 'string') {
+        return done(new Error(`saml profile missing id for key ${profileKeys.id}`))
       }
-
-      if (!user.authentication.authenticationConfiguration.enabled) {
-        log.warn('Failed user login attempt: Authentication ' + user.authentication.authenticationConfiguration.title + ' is disabled.');
-        return next();
+      const idpAccount: IdentityProviderUser = {
+        username: uid,
+        displayName: profile[profileKeys.displayName!] as string,
+        email: profile[profileKeys.email!] as string | undefined,
+        phones: [],
       }
-
-      // DEPRECATED session authorization, remove req.login which creates session in next version
-      req.login(user, function (err) {
-        if (err) {
-          return next(err);
+      const webUser: Pick<Express.User, 'admittingFromIdentityProvider'> = {
+        admittingFromIdentityProvider: {
+          idpName: idp.name,
+          account: idpAccount,
         }
-        AuthenticationInitializer.tokenService.generateToken(user._id.toString(), TokenAssertion.Authorized, 60 * 5)
-          .then(token => {
-            req.token = token;
-            req.user = user;
-            req.info = info
-            next();
-          }).catch(err => {
-            next(err);
-          });
-      });
-    })(req, res, next);
-  }
-
-  AuthenticationInitializer.app.post(
-    `/auth/${strategy.name}/callback`,
-    authenticate,
-    function (req, res) {
-      let state = {};
-      try {
-        state = JSON.parse(req.body.RelayState)
-      } catch (ignore) {
-        console.warn('saml: error parsing RelayState', ignore)
       }
-
-      if (state.initiator === 'mage') {
-        if (state.client === 'mobile') {
-          let uri;
-          if (!req.user.active || !req.user.enabled) {
-            uri = `mage://app/invalid_account?active=${req.user.active}&enabled=${req.user.enabled}`;
-          } else {
-            uri = `mage://app/authentication?token=${req.token}`
-          }
-
-          res.redirect(uri);
-        } else {
-          res.render('authentication', { host: req.getRoot(), login: { token: req.token, user: req.user } });
+      try {
+        const relayState = JSON.parse(req.body.RelayState) || {}
+        if (!relayState) {
+          return done(new Error('missing saml relay state'))
         }
-      } else {
-        if (req.user.active && req.user.enabled) {
-          res.redirect(`/#/signin?strategy=${strategy.name}&action=authorize-device&token=${req.token}`);
-        } else {
-          const action = !req.user.active ? 'inactive-account' : 'disabled-account';
-          res.redirect(`/#/signin?strategy=${strategy.name}&action=${action}`);
+        if (relayState.initiator !== 'mage') {
+          return done(new Error(`invalid saml relay state initiator: ${relayState.initiator}`))
         }
+        webUser.admittingFromIdentityProvider!.flowState = relayState.flowState
       }
-    }
-  );
-}
-
-function setDefaults(strategy) {
-  if (!strategy.settings.profile) {
-    strategy.settings.profile = {};
-  }
-  if (!strategy.settings.profile.displayName) {
-    strategy.settings.profile.displayName = 'email';
-  }
-  if (!strategy.settings.profile.email) {
-    strategy.settings.profile.email = 'email';
-  }
-  if (!strategy.settings.profile.id) {
-    strategy.settings.profile.id = 'uid';
+      catch (err) {
+        return done(err as Error)
+      }
+      done(null, webUser)
+    }) as VerifyWithRequest,
+    (function samlSignOut() {
+      console.warn('saml sign out unimplemented')
+    }) as VerifyWithRequest
+  )
+  const handleIngressFlowRequest = express.Router()
+    .post('/callback',
+      passport.authenticate(samlStrategy),
+    )
+  return {
+    ingressResponseType: IngressResponseType.Redirect,
+    beginIngressFlow(req, res, next, flowState): any {
+      const RelayState = JSON.stringify({ initiator: 'mage', flowState })
+      passport.authenticate(samlStrategy, { additionalParams: { RelayState } } as any)(req, res, next)
+    },
+    handleIngressFlowRequest
   }
-}
-
-function initialize(strategy) {
-  const app = AuthenticationInitializer.app;
-  const passport = AuthenticationInitializer.passport;
-  // const provision = AuthenticationInitializer.provision;
-
-  setDefaults(strategy);
-  configure(strategy);
-
-  // function parseLoginMetadata(req, res, next) {
-  //   req.loginOptions = {
-  //     userAgent: req.headers['user-agent'],
-  //     appVersion: req.param('appVersion')
-  //   };
-
-  //   next();
-  // }
-  app.get(
-    '/auth/' + strategy.name + '/signin',
-    function (req, res, next) {
-      const state = {
-        initiator: 'mage',
-        client: req.query.state
-      };
-
-      passport.authenticate(strategy.name, {
-        additionalParams: { RelayState: JSON.stringify(state) }
-      })(req, res, next);
-    }
-  );
-}
-
-module.exports = {
-  initialize
 }
\ No newline at end of file

From 57f8bb2f830bf910ea56f24a08994a2f3d059564 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 14 Nov 2024 23:33:34 -0700
Subject: [PATCH 155/183] refactor(service): users/auth: rename some session
 repository methods for consistency

---
 .../ingress/ingress.adapters.db.mongoose.ts   |  4 ++--
 service/src/ingress/ingress.entities.ts       | 23 +++++++++++++++----
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/service/src/ingress/ingress.adapters.db.mongoose.ts b/service/src/ingress/ingress.adapters.db.mongoose.ts
index b31d97e98..5bbea9597 100644
--- a/service/src/ingress/ingress.adapters.db.mongoose.ts
+++ b/service/src/ingress/ingress.adapters.db.mongoose.ts
@@ -157,9 +157,9 @@ export class UserIngressBindingsMongooseRepository implements UserIngressBinding
 
   constructor(readonly model: UserIngressBindingsModel) {}
 
-  async readBindingsForUser(userId: UserId): Promise<UserIngressBindings> {
+  async readBindingsForUser(userId: UserId): Promise<UserIngressBindings | null> {
     const doc = await this.model.findById(userId, null, { lean: true })
-    return { userId, bindingsByIdp: new Map(Object.entries(doc?.bindings || {})) }
+    return doc ? { userId, bindingsByIdp: new Map(Object.entries(doc?.bindings || {})) } : null
   }
 
   async readAllBindingsForIdp(idpId: IdentityProviderId, paging?: PagingParameters | undefined): Promise<PageOf<UserIngressBindings>> {
diff --git a/service/src/ingress/ingress.entities.ts b/service/src/ingress/ingress.entities.ts
index dadf3b6c7..6a4dc3d91 100644
--- a/service/src/ingress/ingress.entities.ts
+++ b/service/src/ingress/ingress.entities.ts
@@ -19,11 +19,11 @@ export type SessionExpanded = Omit<Session, 'user' | 'device'> & {
 }
 
 export interface SessionRepository {
-  findSessionByToken(token: string): Promise<Session | null>
+  readSessionByToken(token: string): Promise<Session | null>
   createOrRefreshSession(userId: UserId, deviceId?: string): Promise<Session>
-  removeSession(token: string): Promise<Session | null>
-  removeSessionsForUser(userId: UserId): Promise<number>
-  removeSessionsForDevice(deviceId: DeviceId): Promise<number>
+  deleteSession(token: string): Promise<Session | null>
+  deleteSessionsForUser(userId: UserId): Promise<number>
+  deleteSessionsForDevice(deviceId: DeviceId): Promise<number>
 }
 
 /**
@@ -82,6 +82,10 @@ export interface DeviceEnrollmentPolicy {
  * The identity provider user is the result of mapping a specific IDP account to a Mage user account.
  */
 export type IdentityProviderUser = Pick<User, 'username' | 'displayName' | 'email' | 'phones'>
+  & {
+    idpAccountId?: string
+    idpAccountAttrs?: Record<string, any>
+  }
 
 /**
  * A user ingress binding is the bridge between a Mage user and an identity provider account.  When a user attempts
@@ -90,6 +94,8 @@ export type IdentityProviderUser = Pick<User, 'username' | 'displayName' | 'emai
  */
 export interface UserIngressBinding {
   idpId: IdentityProviderId
+  created: Date
+  updated: Date
   // TODO: evaluate for utility of disabling a single ingress/idp path for a user as opposed to the entire account
   // verified: boolean
   // enabled: boolean
@@ -124,8 +130,15 @@ export interface IdentityProviderRepository {
 }
 
 export interface UserIngressBindingsRepository {
-  readBindingsForUser(userId: UserId): Promise<UserIngressBindings>
+  /**
+   * Return null if the user has no persisted bindings entry.
+   */
+  readBindingsForUser(userId: UserId): Promise<UserIngressBindings | null>
   readAllBindingsForIdp(idpId: IdentityProviderId, paging?: PagingParameters): Promise<PageOf<UserIngressBindings>>
+  /**
+   * Save the given ingress binding to the bindings dictionary for the given user, creating or updating as necessary.
+   * Return the modified ingress bindings.
+   */
   saveUserIngressBinding(userId: UserId, binding: UserIngressBinding): Promise<UserIngressBindings | Error>
   /**
    * Return the binding that was deleted, or null if the user did not have a binding to the given IDP.

From 8d92ae4010e8a3ff15822c34d998b9107808688d Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 21 Nov 2024 19:27:11 -0700
Subject: [PATCH 156/183] refactor(service): users/auth: refactor some app
 level logic to internal/domain services for reuse

---
 .../ingress/ingress.adapters.db.mongoose.ts   |  6 +-
 service/src/ingress/ingress.app.api.ts        |  2 +-
 service/src/ingress/ingress.app.impl.ts       | 48 +++++---------
 service/src/ingress/ingress.entities.ts       | 35 +++++++---
 service/src/ingress/ingress.services.api.ts   | 37 ++++++++++-
 service/src/ingress/ingress.services.impl.ts  | 66 +++++++++++++++++--
 service/src/ingress/local-idp.app.impl.ts     |  2 +-
 7 files changed, 141 insertions(+), 55 deletions(-)

diff --git a/service/src/ingress/ingress.adapters.db.mongoose.ts b/service/src/ingress/ingress.adapters.db.mongoose.ts
index 5bbea9597..4cc7b9968 100644
--- a/service/src/ingress/ingress.adapters.db.mongoose.ts
+++ b/service/src/ingress/ingress.adapters.db.mongoose.ts
@@ -157,9 +157,9 @@ export class UserIngressBindingsMongooseRepository implements UserIngressBinding
 
   constructor(readonly model: UserIngressBindingsModel) {}
 
-  async readBindingsForUser(userId: UserId): Promise<UserIngressBindings | null> {
+  async readBindingsForUser(userId: UserId): Promise<UserIngressBindings> {
     const doc = await this.model.findById(userId, null, { lean: true })
-    return doc ? { userId, bindingsByIdp: new Map(Object.entries(doc?.bindings || {})) } : null
+    return { userId, bindingsByIdpId: new Map(Object.entries(doc?.bindings || {})) }
   }
 
   async readAllBindingsForIdp(idpId: IdentityProviderId, paging?: PagingParameters | undefined): Promise<PageOf<UserIngressBindings>> {
@@ -170,7 +170,7 @@ export class UserIngressBindingsMongooseRepository implements UserIngressBinding
     const _id = new ObjectId(userId)
     const bindingsUpdate = { $set: { [`bindings.${binding.idpId}`]: binding } }
     const doc = await this.model.findOneAndUpdate({ _id }, bindingsUpdate, { upsert: true, new: true })
-    return { userId, bindingsByIdp: new Map(Object.entries(doc.bindings)) }
+    return { userId, bindingsByIdpId: new Map(Object.entries(doc.bindings)) }
   }
 
   async deleteBinding(userId: UserId, idpId: IdentityProviderId): Promise<UserIngressBinding | null> {
diff --git a/service/src/ingress/ingress.app.api.ts b/service/src/ingress/ingress.app.api.ts
index c926c9e42..1889b833a 100644
--- a/service/src/ingress/ingress.app.api.ts
+++ b/service/src/ingress/ingress.app.api.ts
@@ -41,7 +41,7 @@ export interface AdmitFromIdentityProviderRequest {
 }
 
 export interface AdmitFromIdentityProviderResult {
-  mageAccount: User
+  mageAccount: UserExpanded
   admissionToken: string
 }
 
diff --git a/service/src/ingress/ingress.app.impl.ts b/service/src/ingress/ingress.app.impl.ts
index ae46cd297..6233cc26e 100644
--- a/service/src/ingress/ingress.app.impl.ts
+++ b/service/src/ingress/ingress.app.impl.ts
@@ -1,9 +1,8 @@
 import { entityNotFound, infrastructureError } from '../app.api/app.api.errors'
 import { AppResponse } from '../app.api/app.api.global'
-import { UserRepository } from '../entities/users/entities.users'
 import { AdmitFromIdentityProviderOperation, AdmitFromIdentityProviderRequest, authenticationFailedError, EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
-import { IdentityProviderRepository, IdentityProviderUser, UserIngressBindingsRepository } from './ingress.entities'
-import { EnrollNewUser } from './ingress.services.api'
+import { IdentityProviderRepository, IdentityProviderUser } from './ingress.entities'
+import { AdmissionDeniedReason, AdmitUserFromIdentityProviderAccount, EnrollNewUser } from './ingress.services.api'
 import { LocalIdpCreateAccountOperation } from './local-idp.app.api'
 import { JWTService, TokenAssertion } from './verification'
 
@@ -32,12 +31,13 @@ export function CreateEnrollMyselfOperation(createLocalIdpAccount: LocalIdpCreat
     }
     const enrollmentResult = await enrollNewUser(candidateMageAccount, localIdp)
 
+
     // TODO: auto-activate account after enrollment policy
     throw new Error('unimplemented')
   }
 }
 
-export function CreateAdmitFromIdentityProviderOperation(idpRepo: IdentityProviderRepository, ingressBindingRepo: UserIngressBindingsRepository, userRepo: UserRepository, enrollNewUser: EnrollNewUser, tokenService: JWTService): AdmitFromIdentityProviderOperation {
+export function CreateAdmitFromIdentityProviderOperation(idpRepo: IdentityProviderRepository, admitFromIdpAccount: AdmitUserFromIdentityProviderAccount, tokenService: JWTService): AdmitFromIdentityProviderOperation {
   return async function admitFromIdentityProvider(req: AdmitFromIdentityProviderRequest): ReturnType<AdmitFromIdentityProviderOperation> {
     const idp = await idpRepo.findIdpByName(req.identityProviderName)
     if (!idp) {
@@ -45,42 +45,24 @@ export function CreateAdmitFromIdentityProviderOperation(idpRepo: IdentityProvid
     }
     const idpAccount = req.identityProviderUser
     console.info(`admitting user ${idpAccount.username} from identity provider ${idp.name}`)
-    const mageAccount = await userRepo.findByUsername(idpAccount.username)
-      .then(existingAccount => {
-        if (existingAccount) {
-          return ingressBindingRepo.readBindingsForUser(existingAccount.id).then(ingressBindings => {
-            return { mageAccount: existingAccount, ingressBindings }
-          })
+    const admission = await admitFromIdpAccount(idpAccount, idp)
+    if (admission.action === 'denied') {
+      if (admission.mageAccount) {
+        if (admission.reason === AdmissionDeniedReason.PendingApproval) {
+          return AppResponse.error(authenticationFailedError(admission.mageAccount.username, idp.name, 'Your account requires approval from a Mage administrator.'))
         }
-        return enrollNewUser(idpAccount, idp)
-      })
-      .then(enrolled => {
-        const { mageAccount, ingressBindings } = enrolled
-        if (ingressBindings.bindingsByIdp.has(idp.id)) {
-          return mageAccount
+        if (admission.reason === AdmissionDeniedReason.Disabled) {
+          return AppResponse.error(authenticationFailedError(admission.mageAccount.username, idp.name, 'Your account is disabled.'))
         }
-        console.error(`user ${mageAccount.username} has no ingress binding to identity provider ${idp.name}`)
-        return null
-      })
-      .catch(err => {
-        console.error(`error creating user account ${idpAccount.username} from identity provider ${idp.name}`, err)
-        return null
-      })
-    if (!mageAccount) {
+      }
       return AppResponse.error(authenticationFailedError(idpAccount.username, idp.name))
     }
-    if (!mageAccount.active) {
-      return AppResponse.error(authenticationFailedError(mageAccount.username, idp.name, 'Your account requires approval from a Mage administrator.'))
-    }
-    if (!mageAccount.enabled) {
-      return AppResponse.error(authenticationFailedError(mageAccount.username, idp.name, 'Your account is disabled.'))
-    }
     try {
-      const admissionToken = await tokenService.generateToken(mageAccount.id, TokenAssertion.Authenticated, 5 * 60)
-      return AppResponse.success({ mageAccount, admissionToken })
+      const admissionToken = await tokenService.generateToken(admission.mageAccount.id, TokenAssertion.Authenticated, 5 * 60)
+      return AppResponse.success({ mageAccount: admission.mageAccount, admissionToken })
     }
     catch (err) {
-      console.error(`error generating admission token while authenticating user ${mageAccount.username}`, err)
+      console.error(`error generating admission token while authenticating user ${admission.mageAccount.username}`, err)
       return AppResponse.error(infrastructureError('An unexpected error occurred while generating an authentication token.'))
     }
   }
diff --git a/service/src/ingress/ingress.entities.ts b/service/src/ingress/ingress.entities.ts
index 6a4dc3d91..46c284646 100644
--- a/service/src/ingress/ingress.entities.ts
+++ b/service/src/ingress/ingress.entities.ts
@@ -105,15 +105,13 @@ export interface UserIngressBinding {
    * multiple ingress bindings for different identity providers with different account identifiers.
    */
   idpAccountId?: string
-  /**
-   * Any attributes the identity provider or protocol needs to persist about the account mapping
-   */
-  idpAccountAttrs?: Record<string, any>
+  // TODO: unused for now
+  // idpAccountAttrs?: Record<string, any>
 }
 
 export type UserIngressBindings = {
   userId: UserId
-  bindingsByIdp: Map<IdentityProviderId, UserIngressBinding>
+  bindingsByIdpId: Map<IdentityProviderId, UserIngressBinding>
 }
 
 export type IdentityProviderMutableAttrs = Omit<IdentityProvider, 'id' | 'name' | 'protocol'>
@@ -133,7 +131,7 @@ export interface UserIngressBindingsRepository {
   /**
    * Return null if the user has no persisted bindings entry.
    */
-  readBindingsForUser(userId: UserId): Promise<UserIngressBindings | null>
+  readBindingsForUser(userId: UserId): Promise<UserIngressBindings>
   readAllBindingsForIdp(idpId: IdentityProviderId, paging?: PagingParameters): Promise<PageOf<UserIngressBindings>>
   /**
    * Save the given ingress binding to the bindings dictionary for the given user, creating or updating as necessary.
@@ -154,10 +152,29 @@ export interface UserIngressBindingsRepository {
   deleteAllBindingsForIdp(idpId: IdentityProviderId): Promise<number>
 }
 
+export type AdmissionAction =
+  | { admitNew: UserIngressBinding, admitExisting: false, deny: false }
+  | { admitExisting: UserIngressBinding, admitNew: false, deny: false }
+  | { deny: true, admitNew: false, admitExisting: false }
+
+export function determinUserIngressBindingAdmission(idpAccount: IdentityProviderUser, idp: IdentityProvider, bindings: UserIngressBindings): AdmissionAction {
+  if (bindings.bindingsByIdpId.size === 0) {
+    // new user account
+    const now = new Date(Date.now())
+    return { admitNew: { created: now, updated: now, idpId: idp.id, idpAccountId: idpAccount.idpAccountId }, admitExisting: false, deny: false }
+  }
+  const binding = bindings.bindingsByIdpId.get(idp.id)
+  if (binding) {
+    // existing account bound to idp
+    return { admitExisting: binding, admitNew: false, deny: false }
+  }
+  return { deny: true, admitNew: false, admitExisting: false }
+}
+
 /**
- * Return a new user object from the given identity provider account information suitable to persist as newly enrolled
- * user.  The enrollment policy for the identity provider determines the `active` flag and assigned role for the new
- * user.
+ * Return a new user object from the given identity provider account information suitable to persist as a newly
+ * enrolled user.  The enrollment policy for the identity provider determines the `active` flag and assigned role for
+ * the new user.
  */
 export function createEnrollmentCandidateUser(idpAccount: IdentityProviderUser, idp: IdentityProvider): Omit<User, 'id'> {
   const policy = idp.userEnrollmentPolicy
diff --git a/service/src/ingress/ingress.services.api.ts b/service/src/ingress/ingress.services.api.ts
index 63315f1c0..a3ad3c8a6 100644
--- a/service/src/ingress/ingress.services.api.ts
+++ b/service/src/ingress/ingress.services.api.ts
@@ -1,6 +1,39 @@
-import { User } from '../entities/users/entities.users'
+import { UserExpanded } from '../entities/users/entities.users'
 import { IdentityProvider, IdentityProviderUser, UserIngressBindings } from './ingress.entities'
 
+export type AdmissionResult =
+  | {
+    /**
+     * `'admitted'` if the user account is valid for admission and access to Mage, `'denied'` otherwise
+     */
+    action: 'admitted',
+    /**
+     * The existing or newly enrolled Mage account
+     */
+    mageAccount: UserExpanded,
+    /**
+     * Whether the admission resulted in a new Mage account enrollment
+     */
+    enrolled: boolean,
+  }
+  | {
+    action: 'denied',
+    reason: AdmissionDeniedReason,
+    mageAccount: UserExpanded | null,
+    enrolled: boolean,
+  }
+
+export enum AdmissionDeniedReason {
+  PendingApproval = 'PendingApproval',
+  Disabled = 'Disabled',
+  NameConflict = 'NameConflict',
+  InternalError = 'InternalError',
+}
+
+export interface AdmitUserFromIdentityProviderAccount {
+  (idpAccount: IdentityProviderUser, idp: IdentityProvider): Promise<AdmissionResult>
+}
+
 export interface EnrollNewUser {
-  (idpAccount: IdentityProviderUser, idp: IdentityProvider): Promise<{ mageAccount: User, ingressBindings: UserIngressBindings }>
+  (idpAccount: IdentityProviderUser, idp: IdentityProvider): Promise<{ mageAccount: UserExpanded, ingressBindings: UserIngressBindings }>
 }
\ No newline at end of file
diff --git a/service/src/ingress/ingress.services.impl.ts b/service/src/ingress/ingress.services.impl.ts
index 540aec846..0c0edadcb 100644
--- a/service/src/ingress/ingress.services.impl.ts
+++ b/service/src/ingress/ingress.services.impl.ts
@@ -1,8 +1,8 @@
 import { MageEventId } from '../entities/events/entities.events'
 import { Team, TeamId } from '../entities/teams/entities.teams'
-import { User, UserId, UserRepository, UserRepositoryError } from '../entities/users/entities.users'
-import { createEnrollmentCandidateUser, IdentityProvider, IdentityProviderUser, UserIngressBindingsRepository, UserIngressBindings } from './ingress.entities'
-import { EnrollNewUser } from './ingress.services.api'
+import { UserExpanded, UserId, UserRepository, UserRepositoryError } from '../entities/users/entities.users'
+import { createEnrollmentCandidateUser, IdentityProvider, IdentityProviderUser, UserIngressBindingsRepository, UserIngressBindings, determinUserIngressBindingAdmission } from './ingress.entities'
+import { AdmissionDeniedReason, AdmissionResult, AdmitUserFromIdentityProviderAccount, EnrollNewUser } from './ingress.services.api'
 
 export interface AssignTeamMember {
   (member: UserId, team: TeamId): Promise<boolean>
@@ -12,20 +12,74 @@ export interface FindEventTeam {
   (mageEventId: MageEventId): Promise<Team | null>
 }
 
-export function CreateProcessNewUserEnrollmentService(userRepo: UserRepository, ingressBindingRepo: UserIngressBindingsRepository, findEventTeam: FindEventTeam, assignTeamMember: AssignTeamMember): EnrollNewUser {
-  return async function processNewUserEnrollment(idpAccount: IdentityProviderUser, idp: IdentityProvider): Promise<{ mageAccount: User, ingressBindings: UserIngressBindings }> {
+export function CreateUserAdmissionService(userRepo: UserRepository, ingressBindingRepo: UserIngressBindingsRepository, enrollNewUser: EnrollNewUser): AdmitUserFromIdentityProviderAccount {
+  return async function(idpAccount: IdentityProviderUser, idp: IdentityProvider): Promise<AdmissionResult> {
+    return userRepo.findByUsername(idpAccount.username)
+      .then(existingAccount => {
+        if (existingAccount) {
+          return ingressBindingRepo.readBindingsForUser(existingAccount.id).then(ingressBindings => {
+            return { enrolled: false, mageAccount: existingAccount, ingressBindings }
+          })
+        }
+        console.info(`enrolling new user account ${idpAccount.username} from identity provider ${idp.name}`)
+        return enrollNewUser(idpAccount, idp).then(enrollment => ({ enrolled: true, ...enrollment }))
+      })
+      .then<AdmissionResult>(userIngress => {
+        const { enrolled, mageAccount, ingressBindings } = userIngress
+        const idpAdmission = determinUserIngressBindingAdmission(idpAccount, idp, ingressBindings)
+        if (idpAdmission.deny) {
+          console.error(`user ${mageAccount.username} has no ingress binding to identity provider ${idp.name}`)
+          return { action: 'denied', reason: AdmissionDeniedReason.NameConflict, enrolled, mageAccount }
+        }
+        if (idpAdmission.admitNew) {
+          return ingressBindingRepo.saveUserIngressBinding(mageAccount.id, idpAdmission.admitNew)
+            .then<AdmissionResult>(() => ({ action: 'admitted', mageAccount, enrolled }))
+            .catch(err => {
+              console.error(`error saving ingress binding for user ${mageAccount.username} to idp ${idp.name}`, err)
+              return { action: 'denied', reason: AdmissionDeniedReason.InternalError, mageAccount, enrolled }
+            })
+        }
+        return { action: 'admitted', mageAccount, enrolled }
+      })
+      .then<AdmissionResult>(userIngress => {
+        const { action, mageAccount, enrolled } = userIngress
+        if (!mageAccount) {
+          return { action: 'denied', reason: AdmissionDeniedReason.InternalError, mageAccount, enrolled }
+        }
+        if (action === 'denied') {
+          return userIngress
+        }
+        if (!mageAccount.active) {
+          return { action: 'denied', reason: AdmissionDeniedReason.PendingApproval, mageAccount, enrolled }
+        }
+        if (!mageAccount.enabled) {
+          return { action: 'denied', reason: AdmissionDeniedReason.Disabled, mageAccount, enrolled }
+        }
+        return userIngress
+      })
+      .catch<AdmissionResult>(err => {
+        console.error(`error admitting user account ${idpAccount.username} from identity provider ${idp.name}`, err)
+        return { action: 'denied', reason: AdmissionDeniedReason.InternalError, enrolled: false, mageAccount: null }
+      })
+  }
+}
+
+export function CreateNewUserEnrollmentService(userRepo: UserRepository, ingressBindingRepo: UserIngressBindingsRepository, findEventTeam: FindEventTeam, assignTeamMember: AssignTeamMember): EnrollNewUser {
+  return async function processNewUserEnrollment(idpAccount: IdentityProviderUser, idp: IdentityProvider): Promise<{ mageAccount: UserExpanded, ingressBindings: UserIngressBindings }> {
     console.info(`enrolling new user account ${idpAccount.username} from identity provider ${idp.name}`)
     const candidate = createEnrollmentCandidateUser(idpAccount, idp)
     const mageAccount = await userRepo.create(candidate)
     if (mageAccount instanceof UserRepositoryError) {
       throw mageAccount
     }
+    const now = new Date()
     const ingressBindings = await ingressBindingRepo.saveUserIngressBinding(
       mageAccount.id,
       {
         idpId: idp.id,
         idpAccountId: idpAccount.username,
-        idpAccountAttrs: {},
+        created: now,
+        updated: now,
       }
     )
     if (ingressBindings instanceof Error) {
diff --git a/service/src/ingress/local-idp.app.impl.ts b/service/src/ingress/local-idp.app.impl.ts
index c35807235..5a99c4520 100644
--- a/service/src/ingress/local-idp.app.impl.ts
+++ b/service/src/ingress/local-idp.app.impl.ts
@@ -36,7 +36,7 @@ export function CreateLocalIdpCreateAccountOperation(repo: LocalIdpRepository):
     const createdAccount = await repo.createLocalAccount(candidateAccount)
     if (createdAccount instanceof LocalIdpError) {
       if (createdAccount instanceof LocalIdpDuplicateUsernameError) {
-        console.info(`attempted to create local account with duplicate username ${req.username}`)
+        console.error(`attempted to create local account with duplicate username ${req.username}`, createdAccount)
       }
       return AppResponse.error(invalidInput(`Failed to create account ${req.username}.`))
     }

From d5c42f10a1765728d0aa71536b665a14cb782a80 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Thu, 21 Nov 2024 19:28:01 -0700
Subject: [PATCH 157/183] refactor(web-app): users/auth: fix renamed property
 on saml settings ui

---
 .../admin-authentication-saml.component.html                    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web-app/src/app/admin/admin-authentication/admin-authentication-saml/admin-authentication-saml.component.html b/web-app/src/app/admin/admin-authentication/admin-authentication-saml/admin-authentication-saml.component.html
index af132dcfe..c486279f7 100644
--- a/web-app/src/app/admin/admin-authentication/admin-authentication-saml/admin-authentication-saml.component.html
+++ b/web-app/src/app/admin/admin-authentication/admin-authentication-saml/admin-authentication-saml.component.html
@@ -203,7 +203,7 @@
 
   <mat-form-field appearance="outline">
     <mat-label>RAC Comparison</mat-label>
-    <mat-select [(ngModel)]="strategy.settings.RACComparison" (selectionChange)="setDirty(true)">
+    <mat-select [(ngModel)]="strategy.settings.racComparison" (selectionChange)="setDirty(true)">
       <mat-option *ngFor="let rac of racs" [value]="rac.value">
         {{rac.viewValue}}
       </mat-option>

From 54d4f593820a357b9f4eb79c89430fbede87f1b7 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 22 Nov 2024 09:31:12 -0700
Subject: [PATCH 158/183] refactor(service): users/auth: remove optional flag
 on request idp user account

---
 service/src/ingress/ingress.protocol.bindings.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/ingress/ingress.protocol.bindings.ts b/service/src/ingress/ingress.protocol.bindings.ts
index 52dcf875f..96bfb021a 100644
--- a/service/src/ingress/ingress.protocol.bindings.ts
+++ b/service/src/ingress/ingress.protocol.bindings.ts
@@ -3,7 +3,7 @@ import { IdentityProviderUser } from './ingress.entities'
 
 export type IdentityProviderAdmissionWebUser = {
   idpName: string
-  account: IdentityProviderUser | undefined
+  account: IdentityProviderUser
   flowState?: string | undefined
 }
 

From 8ec36098d6672e4660ef83df7ec2cc9e0de7af41 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 22 Nov 2024 09:32:12 -0700
Subject: [PATCH 159/183] refactor(service): users/auth: migrate ldap protocol
 to new ingress scheme

---
 service/src/ingress/ingress.protocol.ldap.ts | 230 ++++++++-----------
 1 file changed, 97 insertions(+), 133 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.ldap.ts b/service/src/ingress/ingress.protocol.ldap.ts
index 279dca8f4..8b7728bc3 100644
--- a/service/src/ingress/ingress.protocol.ldap.ts
+++ b/service/src/ingress/ingress.protocol.ldap.ts
@@ -1,149 +1,113 @@
-const LdapStrategy = require('passport-ldapauth')
-  , log = require('winston')
-  , User = require('../models/user')
-  , Role = require('../models/role')
-  , TokenAssertion = require('./verification').TokenAssertion
-  , api = require('../api')
-  , userTransformer = require('../transformers/user')
-  , { app, passport, tokenService } = require('./index');
+import passport from 'passport'
+import LdapStrategy from 'passport-ldapauth'
+import { IdentityProvider, IdentityProviderUser } from './ingress.entities'
+import { IngressProtocolWebBinding, IngressResponseType } from './ingress.protocol.bindings'
 
-function configure(strategy) {
-  log.info('Configuring ' + strategy.title + ' authentication');
 
-  passport.use(strategy.name, new LdapStrategy({
-    server: {
-      url: strategy.settings.url,
-      bindDN: strategy.settings.bindDN,
-      bindCredentials: strategy.settings.bindCredentials,
-      searchBase: strategy.settings.searchBase,
-      searchFilter: strategy.settings.searchFilter,
-      searchScope: strategy.settings.searchScope,
-      groupSearchBase: strategy.settings.groupSearchBase,
-      groupSearchFilter: strategy.settings.groupSearchFilter,
-      groupSearchScope: strategy.settings.groupSearchScope,
-      bindProperty: strategy.settings.bindProperty,
-      groupDnProperty: strategy.settings.groupDnProperty
-    }
-  },
-    function (profile, done) {
-      const username = profile[strategy.settings.profile.id ];
-      // TODO: users-next
-      User.getUserByAuthenticationStrategy(strategy.type, username, function (err, user) {
-        if (err) return done(err);
-
-        if (!user) {
-          // Create an account for the user
-          Role.getRole('USER_ROLE', function (err, role) {
-            if (err) return done(err);
+type LdapProfileKeys = {
+  id?: string
+  email?: string
+  displayName?: string
+}
 
-            const user = {
-              username: username,
-              displayName: profile[strategy.settings.profile.displayName],
-              email: profile[strategy.settings.profile.email],
-              active: false,
-              roleId: role._id,
-              authentication: {
-                type: strategy.name,
-                id: username,
-                authenticationConfiguration: {
-                  name: strategy.name
-                }
-              }
-            };
-            // TODO: users-next
-            new api.User().create(user).then(newUser => {
-              if (!newUser.authentication.authenticationConfiguration.enabled) {
-                log.warn(newUser.authentication.authenticationConfiguration.title + " authentication is not enabled");
-                return done(null, newUser, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-              }
-              if (newUser.active) {
-                done(null, newUser);
-              } else {
-                done(null, newUser, { status: 403 });
-              }
-            }).catch(err => done(err));
-          });
-        } else if (!user.authentication.authenticationConfiguration.enabled) {
-          log.warn(user.authentication.authenticationConfiguration.title + " authentication is not enabled");
-          return done(null, user, { message: 'Authentication method is not enabled, please contact a MAGE administrator for assistance.' });
-        } else {
-          return done(null, user);
-        }
-      });
-    })
-  );
+type LdapProtocolSettings = LdapStrategy.Options['server'] & {
+  profile?: LdapProfileKeys
 }
 
-function setDefaults(strategy) {
-  if (!strategy.settings.profile) {
-     strategy.settings.profile = {};
+type ReadyLdapProtocolSettings = Omit<LdapProtocolSettings, 'profile'> & { profile: Required<LdapProfileKeys> }
+
+function copyProtocolSettings(from: LdapProtocolSettings): LdapProtocolSettings {
+  const copy = { ...from }
+  if (copy.profile) {
+    copy.profile = { ...from.profile! }
   }
-  if (!strategy.settings.profile.displayName) {
-     strategy.settings.profile.displayName = 'givenname';
+  return copy
+}
+
+function applyDefaultProtocolSettings(idp: IdentityProvider): ReadyLdapProtocolSettings {
+  const settings = copyProtocolSettings(idp.protocolSettings as LdapProtocolSettings)
+  const profileKeys = settings.profile || {}
+  if (!profileKeys.displayName) {
+    profileKeys.displayName = 'givenname';
   }
-  if (!strategy.settings.profile.email) {
-     strategy.settings.profile.email = 'mail';
+  if (!profileKeys.email) {
+    profileKeys.email = 'mail';
   }
-  if (!strategy.settings.profile.id) {
-     strategy.settings.profile.id = 'cn';
+  if (!profileKeys.id) {
+    profileKeys.id = 'cn';
   }
+  settings.profile = profileKeys
+  return settings as ReadyLdapProtocolSettings
 }
 
-function initialize(strategy) {
-  setDefaults(strategy);
-  configure(strategy);
-
-  const authenticationOptions = {
-    invalidLogonHours: `Not Permitted to login to ${strategy.title} account at this time.`,
-    invalidWorkstation: `Not permited to logon to ${strategy.title} account at this workstation.`,
-    passwordExpired: `${strategy.title} password expired.`,
-    accountDisabled: `${strategy.title} account disabled.`,
-    accountExpired: `${strategy.title} account expired.`,
-    passwordMustChange: `User must reset ${strategy.title} password.`,
-    accountLockedOut: `${strategy.title} user account locked.`,
-    invalidCredentials: `Invalid ${strategy.title} username/password.`
-  };
-
-  app.post(`/auth/${strategy.name}/signin`,
-    function authenticate(req, res, next) {
-      passport.authenticate(strategy.name, authenticationOptions, function (err, user, info = {}) {
-        if (err) return next(err);
-
-        if (!user) {
-          return res.status(401).send(info.message);
-        }
-
-        if (!user.active) {
-          return res.status(info.status || 401).send('User account is not approved, please contact your MAGE administrator to approve your account.');
-        }
-
-        if (!user.enabled) {
-          log.warn('Failed user login attempt: User ' + user.username + ' account is disabled.');
-          return res.status(401).send('Your account has been disabled, please contact a MAGE administrator for assistance.')
-        }
+function strategyOptionsFromProtocolSettings(settings: ReadyLdapProtocolSettings): LdapStrategy.Options {
+  return {
+    server: {
+      url: settings.url,
+      bindDN: settings.bindDN,
+      bindCredentials: settings.bindCredentials,
+      searchBase: settings.searchBase,
+      searchFilter: settings.searchFilter,
+      searchScope: settings.searchScope,
+      groupSearchBase: settings.groupSearchBase,
+      groupSearchFilter: settings.groupSearchFilter,
+      groupSearchScope: settings.groupSearchScope,
+      bindProperty: settings.bindProperty,
+      groupDnProperty: settings.groupDnProperty
+    }
+  }
+}
 
-        if (!user.authentication.authenticationConfigurationId) {
-          log.warn('Failed user login attempt: ' + user.authentication.type + ' is not configured');
-          return res.status(401).send(user.authentication.type + ' authentication is not configured, please contact a MAGE administrator for assistance.')
+export function createWebBinding(idp: IdentityProvider, passport: passport.Authenticator): IngressProtocolWebBinding {
+  const settings = applyDefaultProtocolSettings(idp)
+  const profileKeys = settings.profile
+  const strategyOptions = strategyOptionsFromProtocolSettings(settings)
+  const verify: LdapStrategy.VerifyCallback = (profile, done) => {
+    const idpAccount: IdentityProviderUser = {
+      username: profile[profileKeys.id],
+      displayName: profile[profileKeys.displayName],
+      email: profile[profileKeys.email],
+      phones: [],
+      idpAccountId: profile[profileKeys.id]
+    }
+    const webIngressUser: Express.User = {
+      admittingFromIdentityProvider: {
+        account: idpAccount,
+        idpName: idp.name,
+      }
+    }
+    return done(null, webIngressUser)
+  }
+  const title = idp.title
+  const authOptions: LdapStrategy.AuthenticateOptions = {
+    invalidLogonHours: `Access to ${title} account is prohibited at this time.`,
+    invalidWorkstation: `Access to ${title} account is prohibited from this workstation.`,
+    passwordExpired: `${title} password expired.`,
+    accountDisabled: `${title} account disabled.`,
+    accountExpired: `${title} account expired.`,
+    passwordMustChange: `${title} account requires password reset.`,
+    accountLockedOut: `${title} account locked.`,
+    invalidCredentials: `Invalid ${title} credentials.`,
+  }
+  const ldapIdp = new LdapStrategy(strategyOptions, verify)
+  return {
+    ingressResponseType: IngressResponseType.Direct,
+    beginIngressFlow(req, res, next, flowState): any {
+      const completeIngress: passport.AuthenticateCallback = (err, user) => {
+        if (err) {
+          return next(err)
         }
-
-        if (!user.authentication.authenticationConfiguration.enabled) {
-          log.warn('Failed user login attempt: Authentication ' + user.authentication.authenticationConfiguration.title + ' is disabled.');
-          return res.status(401).send(user.authentication.authenticationConfiguration.title + ' authentication is disabled, please contact a MAGE administrator for assistance.')
+        if (user && user.admittingFromIdentityProvider) {
+          user.admittingFromIdentityProvider.flowState = flowState
+          req.user = user
+          return next()
         }
-
-        tokenService.generateToken(user._id.toString(), TokenAssertion.Authorized, 60 * 5)
-          .then(token => {
-            res.json({
-              user: userTransformer.transform(req.user, { path: req.getRoot() }),
-              token: token
-            });
-          }).catch(err => next(err));
-      })(req, res, next);
+        return res.status(500).send('internal server error: invalid ldap ingress state')
+      }
+      passport.authenticate(ldapIdp, authOptions, completeIngress)(req, res, next)
+    },
+    handleIngressFlowRequest(req, res): any {
+      return res.status(400).send('invalid ldap ingress request')
     }
-  );
-};
-
-module.exports = {
-  initialize
+  }
 }
\ No newline at end of file

From 8c0b19fe39dfae8f9efeaeae5aba6354e4b32b40 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 22 Nov 2024 11:30:33 -0700
Subject: [PATCH 160/183] refactor(service): users/auth: fix user reference in
 legacy access utility

---
 service/src/access/index.ts | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/service/src/access/index.ts b/service/src/access/index.ts
index 2dc265e69..8d0a3dd01 100644
--- a/service/src/access/index.ts
+++ b/service/src/access/index.ts
@@ -19,11 +19,10 @@ export = Object.freeze({
    */
   authorize(permission: AnyPermission): express.RequestHandler {
     return function(req, res, next): any {
-      if (req.user?.from !== 'sessionToken') {
+      if (!req.user?.admitted) {
         return next()
       }
-      const role = req.user.account.role
-      const userPermissions = role.permissions
+      const userPermissions = req.user.admitted.account.role.permissions
       if (userPermissions.includes(permission)) {
         return next()
       }

From 396447d2e41be79decdc791da7cabbacbd5aab18 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 22 Nov 2024 11:45:12 -0700
Subject: [PATCH 161/183] refactor(service): users/auth: fix type errors in
 export routes

---
 service/src/models/export.d.ts |  7 ++++---
 service/src/models/export.js   |  2 +-
 service/src/routes/exports.ts  | 35 +++++++++++++++++++---------------
 3 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/service/src/models/export.d.ts b/service/src/models/export.d.ts
index 0fbce07a5..9a5c647f2 100644
--- a/service/src/models/export.d.ts
+++ b/service/src/models/export.d.ts
@@ -1,5 +1,6 @@
 import mongoose from 'mongoose'
 import { MageEventId } from '../entities/events/entities.events'
+import { UserId } from '../entities/users/entities.users'
 import { ExportFormat } from '../export'
 import { ExportOptions } from '../export/exporter'
 import { UserDocument } from './user'
@@ -19,7 +20,7 @@ export type ExportErrorAttrs = {
 }
 
 export type ExportAttrs = {
-  userId: mongoose.Types.ObjectId,
+  userId: UserId,
   relativePath?: string,
   filename?: string,
   exportType: ExportFormat,
@@ -48,8 +49,8 @@ export type PopulateQueryOption = { populate: true }
 export function createExport(spec: Pick<ExportAttrs, 'userId' | 'options' | 'exportType'>): Promise<ExportDocument>
 export function getExportById(id: mongoose.Types.ObjectId | string): Promise<ExportDocument | null>
 export function getExportById(id: mongoose.Types.ObjectId | string, options: PopulateQueryOption): Promise<ExportDocumentPopulated | null>
-export function getExportsByUserId(userId: mongoose.Types.ObjectId): Promise<ExportDocument[]>
-export function getExportsByUserId(userId: mongoose.Types.ObjectId, options: PopulateQueryOption): Promise<ExportDocumentPopulated[]>
+export function getExportsByUserId(userId: UserId): Promise<ExportDocument[]>
+export function getExportsByUserId(userId: UserId, options: PopulateQueryOption): Promise<ExportDocumentPopulated[]>
 export function getExports(): Promise<ExportDocument[]>
 export function getExports(options: PopulateQueryOption): Promise<ExportDocument[]>
 export function count(options?: { filter: any }): Promise<number>
diff --git a/service/src/models/export.js b/service/src/models/export.js
index 1778907c6..dd2ba48de 100644
--- a/service/src/models/export.js
+++ b/service/src/models/export.js
@@ -94,7 +94,7 @@ exports.getExportById = function (id, options = {}) {
 };
 
 exports.getExportsByUserId = function (userId, options = {}) {
-  let query = Export.find({ userId: userId });
+  let query = Export.find({ userId });
   if (options.populate) {
     query = query.populate('userId').populate({ path: 'options.eventId', select: 'name' });
   }
diff --git a/service/src/routes/exports.ts b/service/src/routes/exports.ts
index d4c348839..3ed876fb2 100644
--- a/service/src/routes/exports.ts
+++ b/service/src/routes/exports.ts
@@ -15,11 +15,11 @@ import { EventAccessType } from '../entities/events/entities.events'
 import Export, { ExportDocument } from '../models/export'
 
 type ExportRequest = express.Request & {
-    export?: ExportDocument | null
-    parameters?: {
-      exportId?: ExportDocument['_id']
-      filter: any
-    },
+  export?: ExportDocument | null
+  parameters?: {
+    exportId?: ExportDocument['_id']
+    filter: any
+  },
 }
 
 const DefineExportsRoutes: MageRouteDefinitions = function(app, security) {
@@ -27,12 +27,13 @@ const DefineExportsRoutes: MageRouteDefinitions = function(app, security) {
   const passport = security.authentication.passport;
 
   async function authorizeEventAccess(req: express.Request, res: express.Response, next: express.NextFunction): Promise<void> {
-    if (access.userHasPermission(req.user, ObservationPermission.READ_OBSERVATION_ALL)) {
+    const account = req.user?.admitted?.account
+    if (access.userHasPermission(account, ObservationPermission.READ_OBSERVATION_ALL)) {
       return next();
     }
-    else if (access.userHasPermission(req.user, ObservationPermission.READ_OBSERVATION_EVENT)) {
+    else if (account && access.userHasPermission(account, ObservationPermission.READ_OBSERVATION_EVENT)) {
       // Make sure I am part of this event
-      const allowed = await eventPermissions.userHasEventPermission(req.event!, req.user.id, EventAccessType.Read)
+      const allowed = await eventPermissions.userHasEventPermission(req.event!, account.id, EventAccessType.Read)
       if (allowed) {
         return next();
       }
@@ -43,13 +44,15 @@ const DefineExportsRoutes: MageRouteDefinitions = function(app, security) {
   function authorizeExportAccess(permission: ExportPermission): express.RequestHandler {
     return async function authorizeExportAccess(req, res, next) {
       const exportReq = req as ExportRequest
+      const account = exportReq.user?.admitted?.account
       exportReq.export = await Export.getExportById(req.params.exportId)
-      if (access.userHasPermission(exportReq.user, permission)) {
-        next()
+      if (access.userHasPermission(account, permission)) {
+        return next()
       }
-      else {
-        exportReq.user._id.toString() === exportReq.export?.userId.toString() ? next() : res.sendStatus(403);
+      else if (account && account.id === exportReq.export?.userId.toString()) {
+        return next()
       }
+      res.sendStatus(403)
     }
   }
 
@@ -60,8 +63,9 @@ const DefineExportsRoutes: MageRouteDefinitions = function(app, security) {
     authorizeEventAccess,
     function (req, res, next) {
       const exportReq = req as ExportRequest
+      const userId = exportReq.user!.admitted!.account.id
       const document = {
-        userId: exportReq.user._id,
+        userId,
         exportType: exportReq.body.exportType,
         options: {
           eventId: req.body.eventId,
@@ -97,7 +101,8 @@ const DefineExportsRoutes: MageRouteDefinitions = function(app, security) {
   app.get('/api/exports/myself',
     passport.authenticate('bearer'),
     function (req, res, next) {
-      Export.getExportsByUserId(req.user._id, { populate: true }).then(exports => {
+      const userId = req.user!.admitted!.account.id
+      Export.getExportsByUserId(userId, { populate: true }).then(exports => {
         const response = exportXform.transform(exports, { path: `${req.getRoot()}/api/exports` });
         res.json(response);
       }).catch(err => next(err));
@@ -209,7 +214,7 @@ function parseQueryParams(req: express.Request, res: express.Response, next: exp
     parameters.filter.favorites = String(body.favorites).toLowerCase() === 'true';
     if (parameters.filter.favorites) {
       parameters.filter.favorites = {
-        userId: req.user._id
+        userId: req.user?.admitted?.account.id
       };
     }
     parameters.filter.important = String(body.important).toLowerCase() === 'true';

From 9cc3ea177b10e4d0720bc8178c6f31ae4d03a7f3 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 22 Nov 2024 12:39:46 -0700
Subject: [PATCH 162/183] refactor(service): users/auth: fix type errors in
 export modules and related legacy event and user types

---
 .../adapters.events.controllers.web.legacy.ts |  4 +-
 service/src/entities/users/entities.users.ts  |  6 +--
 service/src/export/exporter.ts                | 15 +++---
 service/src/models/export.d.ts                | 48 +++++++++++--------
 service/src/models/export.js                  | 11 +----
 service/src/models/observation.d.ts           |  6 +--
 service/src/routes/exports.ts                 | 12 ++---
 7 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/service/src/adapters/events/adapters.events.controllers.web.legacy.ts b/service/src/adapters/events/adapters.events.controllers.web.legacy.ts
index 96846a04d..df85e3f66 100644
--- a/service/src/adapters/events/adapters.events.controllers.web.legacy.ts
+++ b/service/src/adapters/events/adapters.events.controllers.web.legacy.ts
@@ -18,7 +18,7 @@ import { UserId } from '../../entities/users/entities.users'
 
 declare module 'express-serve-static-core' {
   export interface Request {
-    event?: EventModel.MageEventDocument
+    event?: EventModel.MageEventModelInstance
     eventEntity?: MageEvent
     access?: { userId: UserId, permission: EventAccessType }
     parameters?: EventQueryParams
@@ -28,7 +28,7 @@ declare module 'express-serve-static-core' {
 }
 
 function determineReadAccess(req: express.Request, res: express.Response, next: express.NextFunction): void {
-  const requestingUser = req.user?.from === 'sessionToken' ? req.user.account : null
+  const requestingUser = req.user?.admitted ? req.user.admitted.account : null
   if (requestingUser && !access.userHasPermission(requestingUser, MageEventPermission.READ_EVENT_ALL)) {
     req.access = { userId: requestingUser.id, permission: EventAccessType.Read }
   }
diff --git a/service/src/entities/users/entities.users.ts b/service/src/entities/users/entities.users.ts
index 824abbe9e..74675ec6d 100644
--- a/service/src/entities/users/entities.users.ts
+++ b/service/src/entities/users/entities.users.ts
@@ -61,13 +61,13 @@ export interface Avatar {
 }
 
 export interface UserRepository {
-  create(userAttrs: Omit<User, 'id' | 'icon' | 'avatar'>): Promise<User | UserRepositoryError>
+  create(userAttrs: Omit<User, 'id' | 'icon' | 'avatar'>): Promise<UserExpanded | UserRepositoryError>
   /**
    * Return `null` if the specified user ID does not exist.
    */
   update(userAttrs: Partial<User> & Pick<User, 'id'>): Promise<User | null | UserRepositoryError>
-  findById(id: UserId): Promise<User | null>
-  findByUsername(username: string): Promise<User | null>
+  findById(id: UserId): Promise<UserExpanded | null>
+  findByUsername(username: string): Promise<UserExpanded | null>
   findAllByIds(ids: UserId[]): Promise<{ [id: string]: User | null }>
   find<MappedResult>(which?: UserFindParameters, mapping?: (user: User) => MappedResult): Promise<PageOf<MappedResult>>
   saveMapIcon(userId: UserId, icon: UserIcon, content: NodeJS.ReadableStream | Buffer): Promise<User | UserRepositoryError>
diff --git a/service/src/export/exporter.ts b/service/src/export/exporter.ts
index 3c6b9ab91..e657f304d 100755
--- a/service/src/export/exporter.ts
+++ b/service/src/export/exporter.ts
@@ -1,12 +1,15 @@
 import mongoose from 'mongoose'
 import * as ObservationModelModule from '../models/observation'
 import * as UserLocationModelModule from '../models/location'
-import { MageEventDocument } from '../models/event'
+import { MageEventModelInstance } from '../models/event'
 import { MageEvent } from '../entities/events/entities.events'
+import { UserId } from '../entities/users/entities.users'
+import { ObservationStateName } from '../entities/observations/entities.observations'
+import { ObservationModelInstance } from '../adapters/observations/adapters.observations.db.mongoose'
 
 
 export interface ExportOptions {
-  event: MageEventDocument
+  event: MageEventModelInstance
   filter: ExportFilter
 }
 
@@ -15,7 +18,7 @@ export interface ExportFilter {
   exportLocations?: boolean
   startDate?: Date
   endDate?: Date
-  favorites?: false | { userId: mongoose.Types.ObjectId }
+  favorites?: false | { userId: UserId }
   important?: boolean
   /**
    * Unintuitively, `attachments: true` will EXCLUDE attachments from the
@@ -29,7 +32,7 @@ export type LocationFetchOptions = Pick<ExportFilter, 'startDate' | 'endDate'>
 
 export class Exporter {
 
-  protected eventDoc: MageEventDocument
+  protected eventDoc: MageEventModelInstance
   protected _event: MageEvent
   protected _filter: ExportFilter
 
@@ -39,10 +42,10 @@ export class Exporter {
     this._filter = options.filter;
   }
 
-  requestObservations(filter: ExportFilter): mongoose.Cursor<ObservationModelModule.ObservationDocument> {
+  requestObservations(filter: ExportFilter): mongoose.Cursor<ObservationModelInstance> {
     const options: ObservationModelModule.ObservationReadStreamOptions = {
       filter: {
-        states: [ 'active' ] as [ 'active' ],
+        states: [ ObservationStateName.Active ],
         observationStartDate: filter.startDate,
         observationEndDate: filter.endDate,
         favorites: filter.favorites,
diff --git a/service/src/models/export.d.ts b/service/src/models/export.d.ts
index 9a5c647f2..9ba770729 100644
--- a/service/src/models/export.d.ts
+++ b/service/src/models/export.d.ts
@@ -19,24 +19,32 @@ export type ExportErrorAttrs = {
   updatedAt: Date,
 }
 
+export type ExportId = string
+
 export type ExportAttrs = {
-  userId: UserId,
-  relativePath?: string,
-  filename?: string,
-  exportType: ExportFormat,
-  status?: ExportStatus,
+  id: ExportId
+  userId: UserId
+  relativePath?: string
+  filename?: string
+  exportType: ExportFormat
+  status?: ExportStatus
   options: {
-    eventId: MageEventId,
+    eventId: MageEventId
     filter: any
   },
-  processingErrors?: ExportErrorAttrs[],
-  expirationDate: Date,
-  lastUpdated: Date,
+  processingErrors?: ExportErrorAttrs[]
+  expirationDate: Date
+  lastUpdated: Date
+}
+
+export type ExportDocument = Omit<ExportAttrs, 'id' | 'userId'> & {
+  _id: mongoose.Types.ObjectId
+  userId: mongoose.Types.ObjectId
 }
 
-export type ExportDocument = ExportAttrs & mongoose.Document
+export type ExportModelInstance = mongoose.HydratedDocument<ExportDocument>
 
-export type ExportDocumentPopulated = Omit<ExportDocument, 'userId' | 'options'> & {
+export type ExportModelInstancePopulated = Omit<ExportModelInstance, 'userId' | 'options'> & {
   // TODO: users-next
   userId: UserDocument | null,
   options: Omit<ExportOptions, 'eventId'> & {
@@ -46,14 +54,14 @@ export type ExportDocumentPopulated = Omit<ExportDocument, 'userId' | 'options'>
 
 export type PopulateQueryOption = { populate: true }
 
-export function createExport(spec: Pick<ExportAttrs, 'userId' | 'options' | 'exportType'>): Promise<ExportDocument>
-export function getExportById(id: mongoose.Types.ObjectId | string): Promise<ExportDocument | null>
-export function getExportById(id: mongoose.Types.ObjectId | string, options: PopulateQueryOption): Promise<ExportDocumentPopulated | null>
-export function getExportsByUserId(userId: UserId): Promise<ExportDocument[]>
-export function getExportsByUserId(userId: UserId, options: PopulateQueryOption): Promise<ExportDocumentPopulated[]>
-export function getExports(): Promise<ExportDocument[]>
-export function getExports(options: PopulateQueryOption): Promise<ExportDocument[]>
+export function createExport(spec: Pick<ExportAttrs, 'userId' | 'options' | 'exportType'>): Promise<ExportModelInstance>
+export function getExportById(id: mongoose.Types.ObjectId | ExportId): Promise<ExportModelInstance | null>
+export function getExportById(id: mongoose.Types.ObjectId | ExportId, options: PopulateQueryOption): Promise<ExportModelInstancePopulated | null>
+export function getExportsByUserId(userId: UserId): Promise<ExportModelInstance[]>
+export function getExportsByUserId(userId: UserId, options: PopulateQueryOption): Promise<ExportModelInstancePopulated[]>
+export function getExports(): Promise<ExportModelInstance[]>
+export function getExports(options: PopulateQueryOption): Promise<ExportModelInstance[]>
 export function count(options?: { filter: any }): Promise<number>
-export function updateExport(id: mongoose.Types.ObjectId, spec: Partial<ExportAttrs>): Promise<ExportDocument | null>
-export function removeExport(id: mongoose.Types.ObjectId | string): Promise<ExportDocument | null>
+export function updateExport(id: ExportId, spec: Partial<ExportAttrs>): Promise<ExportModelInstance | null>
+export function removeExport(id: ExportId): Promise<ExportModelInstance | null>
 
diff --git a/service/src/models/export.js b/service/src/models/export.js
index dd2ba48de..e368d0664 100644
--- a/service/src/models/export.js
+++ b/service/src/models/export.js
@@ -61,9 +61,7 @@ function transform(exportDocument, ret, options) {
   }
 }
 
-ExportSchema.set('toJSON', {
-  transform: transform
-});
+ExportSchema.set('toJSON', { transform });
 
 const Export = mongoose.model('Export', ExportSchema);
 exports.ExportModel = Export;
@@ -80,7 +78,6 @@ exports.createExport = function (data) {
     },
     expirationDate: new Date(Date.now() + exportExpiration)
   });
-
   return Export.create(document);
 };
 
@@ -89,7 +86,6 @@ exports.getExportById = function (id, options = {}) {
   if (options.populate) {
     query = query.populate('userId').populate({ path: 'options.eventId', select: 'name' });
   }
-
   return query.exec()
 };
 
@@ -98,7 +94,6 @@ exports.getExportsByUserId = function (userId, options = {}) {
   if (options.populate) {
     query = query.populate('userId').populate({ path: 'options.eventId', select: 'name' });
   }
-
   return query.exec();
 };
 
@@ -107,22 +102,18 @@ exports.getExports = function (options = {}) {
   if (options.populate) {
     query = query.populate('userId').populate({ path: 'options.eventId', select: 'name' });
   }
-
   return query.exec();
 };
 
 exports.count = function (options) {
   options = options || {};
   const filter = options.filter || {};
-
   const conditions = FilterParser.parse(filter);
-
   return Export.count(conditions).exec();
 };
 
 exports.updateExport = function (id, exp) {
   return Export.findByIdAndUpdate(id, exp, { new: true }).exec();
-
 };
 
 exports.removeExport = function (id) {
diff --git a/service/src/models/observation.d.ts b/service/src/models/observation.d.ts
index 2cfc9e262..a4244c9a5 100644
--- a/service/src/models/observation.d.ts
+++ b/service/src/models/observation.d.ts
@@ -1,7 +1,6 @@
 import { Geometry } from 'geojson'
 import mongoose from 'mongoose'
-import { MageEventAttrs, MageEventId } from '../entities/events/entities.events'
-import { Attachment, FormEntry, ObservationAttrs, ObservationFeatureProperties, ObservationId, ObservationImportantFlag, ObservationState, Thumbnail } from '../entities/observations/entities.observations'
+import { ObservationId, ObservationState } from '../entities/observations/entities.observations'
 import { MageEventDocument } from './event'
 
 export interface ObservationReadOptions {
@@ -12,7 +11,7 @@ export interface ObservationReadOptions {
     observationStartDate?: Date
     observationEndDate?: Date
     states?: ObservationState['name'][]
-    favorites?: false | { userId?: mongoose.Types.ObjectId }
+    favorites?: false | { userId?: UserId }
     important?: boolean
     attachments?: boolean
   }
@@ -28,5 +27,4 @@ export type ObservationReadStreamOptions = Omit<ObservationReadOptions, 'stream'
 }
 export function getObservations(event: MageEventDocument, options: ObservationReadOptions, callback: (err: any, results: ObservationDocument[]) => any): void
 export function getObservations(event: MageEventDocument, options: ObservationReadStreamOptions): mongoose.QueryCursor<ObservationDocument>
-
 export function updateObservation(event: MageEventDocument, observationId: ObservationId, update: any, callback: (err: any | null, obseration: ObservationDocument) => any): void
\ No newline at end of file
diff --git a/service/src/routes/exports.ts b/service/src/routes/exports.ts
index 3ed876fb2..47f850ee1 100644
--- a/service/src/routes/exports.ts
+++ b/service/src/routes/exports.ts
@@ -4,7 +4,7 @@ import express from 'express'
 import fs from 'fs'
 import log from '../logger'
 import { exportDirectory } from '../environment/env'
-import Event, { MageEventDocument } from '../models/event'
+import Event, { MageEventModelInstance } from '../models/event'
 import access from '../access'
 import exportXform from '../transformers/export'
 import { exportFactory, ExportFormat } from '../export'
@@ -12,12 +12,12 @@ import { defaultEventPermissionsService as eventPermissions } from '../permissio
 import { MageRouteDefinitions } from './routes.types'
 import { ExportPermission, ObservationPermission } from '../entities/authorization/entities.permissions'
 import { EventAccessType } from '../entities/events/entities.events'
-import Export, { ExportDocument } from '../models/export'
+import Export, { ExportId, ExportModelInstance } from '../models/export'
 
 type ExportRequest = express.Request & {
-  export?: ExportDocument | null
+  export?: ExportModelInstance | null
   parameters?: {
-    exportId?: ExportDocument['_id']
+    exportId?: ExportModelInstance['_id']
     filter: any
   },
 }
@@ -75,7 +75,7 @@ const DefineExportsRoutes: MageRouteDefinitions = function(app, security) {
       Export.createExport(document).then(result => {
         const response = exportXform.transform(result, { path: req.getPath() });
         res.location(`${req.route.path}/${result._id.toString()}`).status(201).json(response);
-        exportData(result._id, exportReq.event!);
+        exportData(result.id, exportReq.event!);
       })
       .catch(err => next(err))
     }
@@ -240,7 +240,7 @@ function getEvent(req: express.Request, res: express.Response, next: express.Nex
   })
 }
 
-async function exportData(exportId: ExportDocument['_id'], event: MageEventDocument): Promise<void> {
+async function exportData(exportId: ExportId, event: MageEventModelInstance): Promise<void> {
   let exportDocument = await Export.updateExport(exportId, { status: Export.ExportStatus.Running })
   if (!exportDocument) {
     return

From bf4e5c164c705e7fe5a7858f638affad969add75 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 22 Nov 2024 12:40:09 -0700
Subject: [PATCH 163/183] refactor(service): users/auth: delete unused
 anonymous ingress protocol

---
 .../src/ingress/ingress.protocol.anonymous.ts  | 18 ------------------
 1 file changed, 18 deletions(-)
 delete mode 100644 service/src/ingress/ingress.protocol.anonymous.ts

diff --git a/service/src/ingress/ingress.protocol.anonymous.ts b/service/src/ingress/ingress.protocol.anonymous.ts
deleted file mode 100644
index 83ae04a04..000000000
--- a/service/src/ingress/ingress.protocol.anonymous.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-const AuthenticationInitializer = require('./index');
-
-function initialize() {
-  const passport = AuthenticationInitializer.passport;
-
-  const AnonymousStrategy = require('passport-anonymous').Strategy;
-  passport.use(new AnonymousStrategy());
-
-  return {
-    loginStrategy: 'anonymous',
-    authenticationStrategy: 'anonymous',
-    passport: passport
-  };
-};
-
-module.exports = {
-  initialize
-}
\ No newline at end of file

From 04bd2e4ed292bf314ec3885dd74940ce472e9665 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 25 Nov 2024 12:59:45 -0700
Subject: [PATCH 164/183] refactor(service): users/auth: move local idp app
 layer operations to internal services to allow dependency injection and reuse

---
 service/src/ingress/ingress.app.impl.ts       | 27 +++++----
 service/src/ingress/ingress.protocol.local.ts | 19 ++++---
 service/src/ingress/local-idp.app.api.ts      | 12 ----
 service/src/ingress/local-idp.app.impl.ts     | 45 ---------------
 service/src/ingress/local-idp.entities.ts     |  4 ++
 service/src/ingress/local-idp.services.api.ts | 15 +++++
 .../src/ingress/local-idp.services.impl.ts    | 55 +++++++++++++++++++
 7 files changed, 99 insertions(+), 78 deletions(-)
 delete mode 100644 service/src/ingress/local-idp.app.api.ts
 delete mode 100644 service/src/ingress/local-idp.app.impl.ts
 create mode 100644 service/src/ingress/local-idp.services.api.ts
 create mode 100644 service/src/ingress/local-idp.services.impl.ts

diff --git a/service/src/ingress/ingress.app.impl.ts b/service/src/ingress/ingress.app.impl.ts
index 6233cc26e..7cf9af697 100644
--- a/service/src/ingress/ingress.app.impl.ts
+++ b/service/src/ingress/ingress.app.impl.ts
@@ -1,21 +1,25 @@
-import { entityNotFound, infrastructureError } from '../app.api/app.api.errors'
+import { entityNotFound, infrastructureError, invalidInput } from '../app.api/app.api.errors'
 import { AppResponse } from '../app.api/app.api.global'
 import { AdmitFromIdentityProviderOperation, AdmitFromIdentityProviderRequest, authenticationFailedError, EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
 import { IdentityProviderRepository, IdentityProviderUser } from './ingress.entities'
 import { AdmissionDeniedReason, AdmitUserFromIdentityProviderAccount, EnrollNewUser } from './ingress.services.api'
-import { LocalIdpCreateAccountOperation } from './local-idp.app.api'
+import { LocalIdpError, LocalIdpInvalidPasswordError } from './local-idp.entities'
+import { MageLocalIdentityProviderService } from './local-idp.services.api'
 import { JWTService, TokenAssertion } from './verification'
 
 
-export function CreateEnrollMyselfOperation(createLocalIdpAccount: LocalIdpCreateAccountOperation, idpRepo: IdentityProviderRepository, enrollNewUser: EnrollNewUser): EnrollMyselfOperation {
+export function CreateEnrollMyselfOperation(localIdp: MageLocalIdentityProviderService, idpRepo: IdentityProviderRepository, enrollNewUser: EnrollNewUser): EnrollMyselfOperation {
   return async function enrollMyself(req: EnrollMyselfRequest): ReturnType<EnrollMyselfOperation> {
-    const localAccountCreate = await createLocalIdpAccount(req)
-    if (localAccountCreate.error) {
-      return AppResponse.error(localAccountCreate.error)
+    const localIdpAccount = await localIdp.createAccount(req)
+    if (localIdpAccount instanceof LocalIdpError) {
+      if (localIdpAccount instanceof LocalIdpInvalidPasswordError) {
+        return AppResponse.error(invalidInput(localIdpAccount.message))
+      }
+      console.error('error creating local idp account for self-enrollment', localIdpAccount)
+      return AppResponse.error(invalidInput('Error creating local Mage account'))
     }
-    const localAccount = localAccountCreate.success!
     const candidateMageAccount: IdentityProviderUser = {
-      username: localAccount.username,
+      username: localIdpAccount.username,
       displayName: req.displayName,
       phones: [],
     }
@@ -25,12 +29,11 @@ export function CreateEnrollMyselfOperation(createLocalIdpAccount: LocalIdpCreat
     if (req.phone) {
       candidateMageAccount.phones = [ { number: req.phone, type: 'Main' } ]
     }
-    const localIdp = await idpRepo.findIdpByName('local')
-    if (!localIdp) {
+    const idp = await idpRepo.findIdpByName('local')
+    if (!idp) {
       throw new Error('local idp not found')
     }
-    const enrollmentResult = await enrollNewUser(candidateMageAccount, localIdp)
-
+    const enrollmentResult = await enrollNewUser(candidateMageAccount, idp)
 
     // TODO: auto-activate account after enrollment policy
     throw new Error('unimplemented')
diff --git a/service/src/ingress/ingress.protocol.local.ts b/service/src/ingress/ingress.protocol.local.ts
index 05f6ab5b5..d30ad7a05 100644
--- a/service/src/ingress/ingress.protocol.local.ts
+++ b/service/src/ingress/ingress.protocol.local.ts
@@ -3,8 +3,9 @@ import express from 'express'
 import { Strategy as LocalStrategy, VerifyFunction as LocalStrategyVerifyFunction } from 'passport-local'
 import { LocalIdpAccount } from './local-idp.entities'
 import { IdentityProviderUser } from './ingress.entities'
-import { LocalIdpAuthenticateOperation } from './local-idp.app.api'
 import { IngressProtocolWebBinding, IngressResponseType } from './ingress.protocol.bindings'
+import { MageLocalIdentityProviderService } from './local-idp.services.api'
+import { permissionDenied } from '../app.api/app.api.errors'
 
 
 function userForLocalIdpAccount(account: LocalIdpAccount): IdentityProviderUser {
@@ -15,15 +16,15 @@ function userForLocalIdpAccount(account: LocalIdpAccount): IdentityProviderUser
   }
 }
 
-function createLocalStrategy(localIdpAuthenticate: LocalIdpAuthenticateOperation, flowState: string | undefined): passport.Strategy {
+function createLocalStrategy(localIdp: MageLocalIdentityProviderService, flowState: string | undefined): passport.Strategy {
   const verify: LocalStrategyVerifyFunction = async function LocalIngressProtocolVerify(username, password, done) {
-    const authResult = await localIdpAuthenticate({ username, password })
-    if (authResult.success) {
-      const localAccount = authResult.success
-      const localIdpUser = userForLocalIdpAccount(localAccount)
-      return done(null, { admittingFromIdentityProvider: { idpName: 'local', account: localIdpUser, flowState } })
+    const authResult = await localIdp.authenticate({ username, password })
+    if (!authResult || authResult.failed) {
+      return done(permissionDenied('local authentication failed', username))
     }
-    return done(authResult.error)
+    const localAccount = authResult.authenticated
+    const localIdpUser = userForLocalIdpAccount(localAccount)
+    return done(null, { admittingFromIdentityProvider: { idpName: 'local', account: localIdpUser, flowState } })
   }
   return new LocalStrategy(verify)
 }
@@ -40,7 +41,7 @@ const validateSigninRequest: express.RequestHandler = function LocalProtocolIngr
   next()
 }
 
-export function createWebBinding(passport: passport.Authenticator, localIdpAuthenticate: LocalIdpAuthenticateOperation): IngressProtocolWebBinding {
+export function createLocalProtocolWebBinding(passport: passport.Authenticator, localIdpAuthenticate: MageLocalIdentityProviderService): IngressProtocolWebBinding {
   return {
     ingressResponseType: IngressResponseType.Direct,
     beginIngressFlow: (req, res, next, flowState): any => {
diff --git a/service/src/ingress/local-idp.app.api.ts b/service/src/ingress/local-idp.app.api.ts
deleted file mode 100644
index 70ad88862..000000000
--- a/service/src/ingress/local-idp.app.api.ts
+++ /dev/null
@@ -1,12 +0,0 @@
-import { EntityNotFoundError, InvalidInputError } from '../app.api/app.api.errors'
-import { AppResponse } from '../app.api/app.api.global'
-import { LocalIdpAccount, LocalIdpCredentials } from './local-idp.entities'
-
-
-export interface LocalIdpAuthenticateOperation {
-  (req: LocalIdpCredentials): Promise<AppResponse<LocalIdpAccount, EntityNotFoundError | InvalidInputError>>
-}
-
-export interface LocalIdpCreateAccountOperation {
-  (req: LocalIdpCredentials): Promise<AppResponse<LocalIdpAccount, InvalidInputError>>
-}
diff --git a/service/src/ingress/local-idp.app.impl.ts b/service/src/ingress/local-idp.app.impl.ts
deleted file mode 100644
index 5a99c4520..000000000
--- a/service/src/ingress/local-idp.app.impl.ts
+++ /dev/null
@@ -1,45 +0,0 @@
-import { invalidInput } from '../app.api/app.api.errors'
-import { AppResponse } from '../app.api/app.api.global'
-import { LocalIdpAuthenticateOperation, LocalIdpCreateAccountOperation } from './local-idp.app.api'
-import { attemptAuthentication, LocalIdpDuplicateUsernameError, LocalIdpError, LocalIdpInvalidPasswordError, LocalIdpRepository, prepareNewAccount } from './local-idp.entities'
-
-
-export function CreateLocalIdpAuthenticateOperation(repo: LocalIdpRepository): LocalIdpAuthenticateOperation {
-  return async function localIdpAuthenticate(req): ReturnType<LocalIdpAuthenticateOperation> {
-    const account = await repo.readLocalAccount(req.username)
-    if (!account) {
-      console.info('local account does not exist:', req.username)
-      return AppResponse.error(invalidInput(`Failed to authenticate user ${req.username}`))
-    }
-    const securityPolicy = await repo.readSecurityPolicy()
-    const attempt = await attemptAuthentication(account, req.password, securityPolicy.accountLock)
-    if (attempt.failed) {
-      console.info('local authentication failed', attempt.failed)
-      return AppResponse.error(invalidInput(`Failed to authenticate user ${req.username}`))
-    }
-    const accountSaved = await repo.updateLocalAccount(attempt.authenticated)
-    if (accountSaved) {
-      return AppResponse.success(accountSaved)
-    }
-    console.error(`account for username ${req.username} did not exist for update after authentication`)
-    return AppResponse.error(invalidInput(`Failed to authenticate user ${req.username}`))
-  }
-}
-
-export function CreateLocalIdpCreateAccountOperation(repo: LocalIdpRepository): LocalIdpCreateAccountOperation {
-  return async function localIdpCreateAccount(req) {
-    const securityPolicy = await repo.readSecurityPolicy()
-    const candidateAccount = await prepareNewAccount(req.username, req.password, securityPolicy)
-    if (candidateAccount instanceof LocalIdpInvalidPasswordError) {
-      return AppResponse.error(invalidInput(`Failed to create account ${req.username}.`, [ candidateAccount.message, 'password' ]))
-    }
-    const createdAccount = await repo.createLocalAccount(candidateAccount)
-    if (createdAccount instanceof LocalIdpError) {
-      if (createdAccount instanceof LocalIdpDuplicateUsernameError) {
-        console.error(`attempted to create local account with duplicate username ${req.username}`, createdAccount)
-      }
-      return AppResponse.error(invalidInput(`Failed to create account ${req.username}.`))
-    }
-    return AppResponse.success(createdAccount)
-  }
-}
\ No newline at end of file
diff --git a/service/src/ingress/local-idp.entities.ts b/service/src/ingress/local-idp.entities.ts
index d7a75217b..ba9a59790 100644
--- a/service/src/ingress/local-idp.entities.ts
+++ b/service/src/ingress/local-idp.entities.ts
@@ -158,6 +158,10 @@ export class LocalIdpFailedAuthenticationError extends LocalIdpError {
   }
 }
 
+export class LocalIdpAccountNotFoundError extends LocalIdpError {
+
+}
+
 function invalidPasswordError(reason: string): LocalIdpError {
   return new LocalIdpError(reason)
 }
diff --git a/service/src/ingress/local-idp.services.api.ts b/service/src/ingress/local-idp.services.api.ts
new file mode 100644
index 000000000..0a342d4df
--- /dev/null
+++ b/service/src/ingress/local-idp.services.api.ts
@@ -0,0 +1,15 @@
+import { LocalIdpAccount, LocalIdpAuthenticationResult, LocalIdpCredentials, LocalIdpError } from './local-idp.entities'
+
+
+export interface MageLocalIdentityProviderService {
+  createAccount(credentials: LocalIdpCredentials): Promise<LocalIdpAccount | LocalIdpError>
+  /**
+   * Return `null` if no account for the given username exists.
+   */
+  deleteAccount(username: string): Promise<LocalIdpAccount | null>
+  /**
+   * Return `null` if no account for the given username exists.  If authentication fails, update the corresponding
+   * account according to the service's account lock policy.
+   */
+  authenticate(credentials: LocalIdpCredentials): Promise<LocalIdpAuthenticationResult | null>
+}
\ No newline at end of file
diff --git a/service/src/ingress/local-idp.services.impl.ts b/service/src/ingress/local-idp.services.impl.ts
new file mode 100644
index 000000000..8a94e5ff9
--- /dev/null
+++ b/service/src/ingress/local-idp.services.impl.ts
@@ -0,0 +1,55 @@
+import { attemptAuthentication, LocalIdpAccount, LocalIdpAuthenticationResult, LocalIdpCredentials, LocalIdpDuplicateUsernameError, LocalIdpError, LocalIdpInvalidPasswordError, LocalIdpRepository, prepareNewAccount } from './local-idp.entities'
+import { MageLocalIdentityProviderService } from './local-idp.services.api'
+
+
+export function createLocalIdentityProviderService(repo: LocalIdpRepository): MageLocalIdentityProviderService {
+
+  async function createAccount(credentials: LocalIdpCredentials): Promise<LocalIdpAccount | LocalIdpError> {
+    const securityPolicy = await repo.readSecurityPolicy()
+    const { username, password } = credentials
+    const candidateAccount = await prepareNewAccount(username, password, securityPolicy)
+    if (candidateAccount instanceof LocalIdpInvalidPasswordError) {
+      return candidateAccount
+    }
+    const createdAccount = await repo.createLocalAccount(candidateAccount)
+    if (createdAccount instanceof LocalIdpError) {
+      if (createdAccount instanceof LocalIdpDuplicateUsernameError) {
+        console.error(`attempted to create local account with duplicate username ${username}`, createdAccount)
+      }
+      return createdAccount
+    }
+    return createdAccount
+  }
+
+  function deleteAccount(username: string): Promise<LocalIdpAccount | null> {
+    return repo.deleteLocalAccount(username)
+  }
+
+  async function authenticate(credentials: LocalIdpCredentials): Promise<LocalIdpAuthenticationResult | null> {
+    const { username, password } = credentials
+    const account = await repo.readLocalAccount(username)
+    if (!account) {
+      console.info('local account does not exist:', username)
+      return null
+    }
+    const securityPolicy = await repo.readSecurityPolicy()
+    const attempt = await attemptAuthentication(account, password, securityPolicy.accountLock)
+    if (attempt.failed) {
+      console.info('local authentication failed', attempt.failed)
+      return attempt
+    }
+    const accountSaved = await repo.updateLocalAccount(attempt.authenticated)
+    if (accountSaved) {
+      attempt.authenticated = accountSaved
+      return attempt
+    }
+    console.error(`account for username ${username} did not exist for update after authentication attempt`)
+    return null
+  }
+
+  return {
+    createAccount,
+    deleteAccount,
+    authenticate,
+  }
+}

From 606be37090bedf0c61bf83d0834651499d055834 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 25 Nov 2024 13:20:30 -0700
Subject: [PATCH 165/183] refactor(service): users/auth: more specific names
 for ingress protocol binding factory functions

---
 service/src/ingress/ingress.protocol.ldap.ts  | 2 +-
 service/src/ingress/ingress.protocol.oauth.ts | 2 +-
 service/src/ingress/ingress.protocol.oidc.ts  | 2 +-
 service/src/ingress/ingress.protocol.saml.ts  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.ldap.ts b/service/src/ingress/ingress.protocol.ldap.ts
index 8b7728bc3..24c65df0e 100644
--- a/service/src/ingress/ingress.protocol.ldap.ts
+++ b/service/src/ingress/ingress.protocol.ldap.ts
@@ -58,7 +58,7 @@ function strategyOptionsFromProtocolSettings(settings: ReadyLdapProtocolSettings
   }
 }
 
-export function createWebBinding(idp: IdentityProvider, passport: passport.Authenticator): IngressProtocolWebBinding {
+export function createLdapProtocolWebBinding(idp: IdentityProvider, passport: passport.Authenticator): IngressProtocolWebBinding {
   const settings = applyDefaultProtocolSettings(idp)
   const profileKeys = settings.profile
   const strategyOptions = strategyOptionsFromProtocolSettings(settings)
diff --git a/service/src/ingress/ingress.protocol.oauth.ts b/service/src/ingress/ingress.protocol.oauth.ts
index 6d0583b4b..992c9570c 100644
--- a/service/src/ingress/ingress.protocol.oauth.ts
+++ b/service/src/ingress/ingress.protocol.oauth.ts
@@ -88,7 +88,7 @@ type OAuth2Info = { state: string }
  * The `baseUrl` parameter is the URL at which Mage will mount the returned `express.Router`, including any
  * distinguishing component of the given `IdentityProvider`, without a trailing slash, e.g. `/auth/example-idp`.
  */
-export function createWebBinding(idp: IdentityProvider, passport: Authenticator, baseUrl: string): IngressProtocolWebBinding {
+export function createOAuthProtocolWebBinding(idp: IdentityProvider, passport: Authenticator, baseUrl: string): IngressProtocolWebBinding {
   const settings = applyDefaultProtocolSettings(idp)
   const profileURL = settings.profileURL
   const customHeaders = settings.headers?.basic ? {
diff --git a/service/src/ingress/ingress.protocol.oidc.ts b/service/src/ingress/ingress.protocol.oidc.ts
index 50054a82b..29a336b3c 100644
--- a/service/src/ingress/ingress.protocol.oidc.ts
+++ b/service/src/ingress/ingress.protocol.oidc.ts
@@ -52,7 +52,7 @@ function applyDefaultProtocolSettings(idp: IdentityProvider): OpenIdConnectProto
   return settings
 }
 
-export function createWebBinding(idp: IdentityProvider, passport: passport.Authenticator, baseUrl: string): IngressProtocolWebBinding {
+export function createOIDCProtocolWebBinding(idp: IdentityProvider, passport: passport.Authenticator, baseUrl: string): IngressProtocolWebBinding {
   const settings = applyDefaultProtocolSettings(idp)
   const verify: OpenIdConnectStrategy.VerifyFunction = (
     issuer: string,
diff --git a/service/src/ingress/ingress.protocol.saml.ts b/service/src/ingress/ingress.protocol.saml.ts
index 521703152..c4c48c0ab 100644
--- a/service/src/ingress/ingress.protocol.saml.ts
+++ b/service/src/ingress/ingress.protocol.saml.ts
@@ -64,7 +64,7 @@ function applyDefaultProtocolSettings(idp: IdentityProvider): SamlProtocolSettin
   return settings
 }
 
-export function createWebBinding(idp: IdentityProvider, passport: Authenticator, baseUrlPath: string): IngressProtocolWebBinding {
+export function createSamlProtocolWebBinding(idp: IdentityProvider, passport: Authenticator, baseUrlPath: string): IngressProtocolWebBinding {
   const { profile: profileKeys, ...settings } = applyDefaultProtocolSettings(idp)
   // TODO: this will need the the saml callback override change
   settings.path = `${baseUrlPath}/callback`

From af1da7570ca42ac5361e99a9bb2ae75dacaf5b41 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 25 Nov 2024 13:20:50 -0700
Subject: [PATCH 166/183] refactor(service): users/auth: add main module for
 ingress initialization

---
 service/src/ingress/ingress.main.ts | 45 +++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 service/src/ingress/ingress.main.ts

diff --git a/service/src/ingress/ingress.main.ts b/service/src/ingress/ingress.main.ts
new file mode 100644
index 000000000..fec508993
--- /dev/null
+++ b/service/src/ingress/ingress.main.ts
@@ -0,0 +1,45 @@
+import passport from 'passport'
+import { IngressProtocolWebBindingCache } from './ingress.adapters.controllers.web'
+import { IdentityProvider, IdentityProviderRepository } from './ingress.entities'
+import { IngressProtocolWebBinding } from './ingress.protocol.bindings'
+import { createLdapProtocolWebBinding } from './ingress.protocol.ldap'
+import { createLocalProtocolWebBinding } from './ingress.protocol.local'
+import { createOAuthProtocolWebBinding } from './ingress.protocol.oauth'
+import { createOIDCProtocolWebBinding } from './ingress.protocol.oidc'
+import { createSamlProtocolWebBinding } from './ingress.protocol.saml'
+import { MageLocalIdentityProviderService } from './local-idp.services.api'
+
+export function createIdpCache(idpRepo: IdentityProviderRepository, localIdp: MageLocalIdentityProviderService): IngressProtocolWebBindingCache {
+  const bindingsByIdpName = new Map<string, { idp: IdentityProvider, idpBinding: IngressProtocolWebBinding }>()
+  const services: BindingServices = {
+    passport,
+    localIdp,
+  }
+  async function idpWebBindingForIdpName(idpName: string): Promise<{ idp: IdentityProvider, idpBinding: IngressProtocolWebBinding } | null> {
+    const cached = bindingsByIdpName.get(idpName)
+    if (cached) {
+      return cached
+    }
+    const idp = await idpRepo.findIdpByName(idpName)
+    if (!idp) {
+      return null
+    }
+    const idpBinding = createWebBinding(idp, services)
+    const cacheEntry = { idp, idpBinding }
+    bindingsByIdpName.set(idp.name, cacheEntry)
+    return cacheEntry
+  }
+  return { idpWebBindingForIdpName }
+}
+
+type BindingServices = {
+  passport: passport.Authenticator
+  localIdp: MageLocalIdentityProviderService
+}
+
+function createWebBinding(idp: IdentityProvider, services: BindingServices) {
+  if (idp.protocol === 'local') {
+    return createLocalProtocolWebBinding(services.passport, services.localIdp)
+  }
+  throw new Error(`cannot create ingress web binding for idp:\n${JSON.stringify(idp, null, 2)}`)
+}
\ No newline at end of file

From b79a80a15159f0dec532a8da6e7c5b8e45547e41 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Mon, 25 Nov 2024 13:22:03 -0700
Subject: [PATCH 167/183] refactor(service): users/auth: add app operation
 stubs to delete identity provider

---
 service/src/ingress/ingress.app.api.ts | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/service/src/ingress/ingress.app.api.ts b/service/src/ingress/ingress.app.api.ts
index 1889b833a..b96544331 100644
--- a/service/src/ingress/ingress.app.api.ts
+++ b/service/src/ingress/ingress.app.api.ts
@@ -66,6 +66,18 @@ export interface UpdateIdentityProviderOperation {
   (req: UpdateIdentityProviderRequest): Promise<AppResponse<UpdateIdentityProviderResult, PermissionDeniedError | EntityNotFoundError>>
 }
 
+export interface DeleteIdentityProviderRequest {
+
+}
+
+export interface DeleteIdentityProviderResult {
+
+}
+
+export interface DeleteIdentityProviderOperation {
+  (req: DeleteIdentityProviderRequest): Promise<AppResponse<DeleteIdentityProviderResult, PermissionDeniedError | EntityNotFoundError>>
+}
+
 export const ErrAuthenticationFailed = Symbol.for('MageError.Ingress.AuthenticationFailed')
 export type AuthenticationFailedErrorData = { username: string, identityProviderName: string }
 export type AuthenticationFailedError = MageError<typeof ErrAuthenticationFailed, AuthenticationFailedErrorData>

From 695b405dff26595c1d331826e8c43f99921cbe1d Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Wed, 27 Nov 2024 22:08:24 -0700
Subject: [PATCH 168/183] refactor(service): users/auth: migrating user
 repository - implement entity mapping and some interface methods; remove
 authentication id foreign key and unused icon type property from schema

---
 .../users/adapters.users.db.mongoose.ts       | 304 ++++++++++--------
 service/src/entities/users/entities.users.ts  |   5 +-
 2 files changed, 170 insertions(+), 139 deletions(-)

diff --git a/service/src/adapters/users/adapters.users.db.mongoose.ts b/service/src/adapters/users/adapters.users.db.mongoose.ts
index e29d75b8f..351c69253 100644
--- a/service/src/adapters/users/adapters.users.db.mongoose.ts
+++ b/service/src/adapters/users/adapters.users.db.mongoose.ts
@@ -1,48 +1,35 @@
-import { User, UserId, UserRepository, UserFindParameters, UserIcon } from '../../entities/users/entities.users'
+import { User, UserId, UserRepository, UserFindParameters, UserIcon, Avatar, UserRepositoryError, UserRepositoryErrorCode, UserExpanded } from '../../entities/users/entities.users'
 import { BaseMongooseRepository, pageQuery } from '../base/adapters.base.db.mongoose'
 import { PageOf, pageOf } from '../../entities/entities.global'
 import _ from 'lodash'
 import mongoose from 'mongoose'
 import { RoleDocument, RoleJson } from '../../models/role'
-import { Authentication } from '../../entities/authentication/entities.authentication'
+import { ObjectId } from 'bson'
+import { Role } from '../../entities/authorization/entities.authorization'
 
 export const UserModelName = 'User'
 
-export type UserDocumentAttrs = Omit<User, 'authenticationId' | 'id' | 'roleId'> & {
+/**
+ * This is the raw document structure that MongoDB stores, and that the
+ * Mongoose/MongoDB driver retrieves without preforming any join-populate
+ * operation with related models.
+ */
+export type UserDocument = Omit<User, 'id' | 'roleId'> & {
   _id: mongoose.Types.ObjectId,
-  authenticationId: mongoose.Types.ObjectId,
   roleId: mongoose.Types.ObjectId,
 }
 
-export type UserDocument = mongoose.Document<mongoose.Types.ObjectId, any, UserDocumentAttrs> & {
-  // _id: mongoose.Types.ObjectId
-  // id: string
-  // username: string
-  // displayName: string
-  // email?: string
-  // phones: Phone[]
-  // avatar: Avatar
-  // icon: UserIcon
-  // active: boolean
-  // enabled: boolean
-  // roleId: mongoose.Types.ObjectId | RoleDocument
-  // authenticationId: mongoose.Types.ObjectId | mongoose.Document
-  // status?: string
-  // recentEventIds: number[]
-  // createdAt: Date
-  // lastUpdated: Date
-  // toJSON(): UserJson
-}
+export type UserExpandedDocument = Omit<UserDocument, 'roleId'>
+  & { roleId: RoleDocument }
 
-// TODO: this probably needs an update now with new authentication changes
-export type UserJson = Omit<UserDocumentAttrs, '_id' | 'avatar' | 'roleId' | 'authenticationId' | keyof mongoose.Document>
+// TODO: users-next: this probably needs an update now with new authentication changes
+export type UserJson = Omit<UserDocument, '_id' | 'avatar' | 'roleId' | keyof mongoose.Document>
   & {
     id: mongoose.Types.ObjectId,
     icon: Omit<UserIcon, 'relativePath'>,
     avatarUrl?: string,
   }
   & (RolePopulated | RoleReferenced)
-  & (AuthenticationPopulated | AuthenticationReferenced)
 
 type RoleReferenced = {
   roleId: string,
@@ -54,17 +41,9 @@ type RolePopulated = {
   role: RoleJson
 }
 
-type AuthenticationPopulated = {
-  authenticationId: never,
-  authentication: Authentication
-}
-
-type AuthenticationReferenced = {
-  authenticationId: string,
-  authentication: never
-}
-
-export type UserModel = mongoose.Model<UserDocument>
+export type UserModel = mongoose.Model<UserDocument, object, object, object, mongoose.Schema<UserDocument>>
+export type UserModelInstance = mongoose.HydratedDocument<UserDocument>
+export type UserExpandedModelInstance = mongoose.HydratedDocument<UserDocument, { roleId: mongoose.HydratedDocument<RoleDocument> }>
 
 export const UserPhoneSchema = new mongoose.Schema(
   {
@@ -77,7 +56,7 @@ export const UserPhoneSchema = new mongoose.Schema(
   }
 )
 
-export const UserSchema = new mongoose.Schema<UserDocumentAttrs>(
+export const UserSchema = new mongoose.Schema<UserDocument>(
   {
     username: { type: String, required: true, unique: true },
     displayName: { type: String, required: true },
@@ -89,7 +68,6 @@ export const UserSchema = new mongoose.Schema<UserDocumentAttrs>(
       relativePath: { type: String, required: false }
     },
     icon: {
-      type: { type: String, enum: ['none', 'upload', 'create'], default: 'none' },
       text: { type: String },
       color: { type: String },
       contentType: { type: String, required: false },
@@ -100,7 +78,6 @@ export const UserSchema = new mongoose.Schema<UserDocumentAttrs>(
     enabled: { type: Boolean, default: true, required: true },
     roleId: { type: mongoose.Schema.Types.ObjectId, ref: 'Role', required: true },
     recentEventIds: [ { type: Number, ref: 'Event' } ],
-    authenticationId: { type: mongoose.Schema.Types.ObjectId, ref: 'Authentication', required: true }
   },
   {
     versionKey: false,
@@ -116,74 +93,76 @@ export const UserSchema = new mongoose.Schema<UserDocumentAttrs>(
   }
 )
 
-UserSchema.virtual('authentication').get(function () {
-  return this.populated('authenticationId') ? this.authenticationId : null;
-});
+// TODO: users-next: find references and delete
+// UserSchema.virtual('authentication').get(function () {
+//   return this.populated('authenticationId') ? this.authenticationId : null;
+// });
 
 // Lowercase the username we store, this will allow for case insensitive usernames
 // Validate that username does not already exist
-UserSchema.pre('save', function (next) {
-  const user = this;
-  user.username = user.username.toLowerCase();
-  this.model('User').findOne({ username: user.username }, function (err, possibleDuplicate) {
-    if (err) return next(err);
-
-    if (possibleDuplicate && !possibleDuplicate._id.equals(user._id)) {
-      const error = new Error('username already exists');
-      error.status = 409;
-      return next(error);
-    }
-
-    next();
-  });
-});
-
-UserSchema.pre('save', function (next) {
-  const user = this;
-  if (user.active === false || user.enabled === false) {
-    Token.removeTokensForUser(user, function (err) {
-      next(err);
-    });
-  } else {
-    next();
-  }
-});
-
-UserSchema.pre('remove', function (next) {
-  const user = this;
-
-  async.parallel({
-    location: function (done) {
-      Location.removeLocationsForUser(user, done);
-    },
-    cappedlocation: function (done) {
-      CappedLocation.removeLocationsForUser(user, done);
-    },
-    token: function (done) {
-      Token.removeTokensForUser(user, done);
-    },
-    login: function (done) {
-      Login.removeLoginsForUser(user, done);
-    },
-    observation: function (done) {
-      Observation.removeUser(user, done);
-    },
-    eventAcl: function (done) {
-      Event.removeUserFromAllAcls(user, function (err) {
-        done(err);
-      });
-    },
-    teamAcl: function (done) {
-      Team.removeUserFromAllAcls(user, done);
-    },
-    authentication: function (done) {
-      Authentication.removeAuthenticationById(user.authenticationId, done);
-    }
-  },
-    function (err) {
-      next(err);
-    });
-});
+// TODO: users-next: properly set 409 statusj in web layer
+// UserSchema.pre('save', function (next) {
+//   const user = this;
+//   user.username = user.username.toLowerCase();
+//   this.model('User').findOne({ username: user.username }, function (err, possibleDuplicate) {
+//     if (err) return next(err);
+//     if (possibleDuplicate && !possibleDuplicate._id.equals(user._id)) {
+//       const error = new Error('username already exists');
+//       error.status = 409;
+//       return next(error);
+//     }
+//     next();
+//   });
+// });
+
+// TODO: users-next: this is business logic that belongs elsewhere
+// UserSchema.pre('save', function (next) {
+//   const user = this;
+//   if (user.active === false || user.enabled === false) {
+//     Token.removeTokensForUser(user, function (err) {
+//       next(err);
+//     });
+//   } else {
+//     next();
+//   }
+// });
+
+// TODO: users-next: move to app/service layer
+// UserSchema.pre('remove', function (next) {
+//   const user = this;
+
+//   async.parallel({
+//     location: function (done) {
+//       Location.removeLocationsForUser(user, done);
+//     },
+//     cappedlocation: function (done) {
+//       CappedLocation.removeLocationsForUser(user, done);
+//     },
+//     token: function (done) {
+//       Token.removeTokensForUser(user, done);
+//     },
+//     login: function (done) {
+//       Login.removeLoginsForUser(user, done);
+//     },
+//     observation: function (done) {
+//       Observation.removeUser(user, done);
+//     },
+//     eventAcl: function (done) {
+//       Event.removeUserFromAllAcls(user, function (err) {
+//         done(err);
+//       });
+//     },
+//     teamAcl: function (done) {
+//       Team.removeUserFromAllAcls(user, done);
+//     },
+//     authentication: function (done) {
+//       Authentication.removeAuthenticationById(user.authenticationId, done);
+//     }
+//   },
+//     function (err) {
+//       next(err);
+//     });
+// });
 
 // eslint-disable-next-line complexity
 function DbUserToObject(user, userOut, options) {
@@ -216,55 +195,96 @@ function DbUserToObject(user, userOut, options) {
   }
 
   if (user.avatar && user.avatar.relativePath) {
-    // TODO, don't really like this, need a better way to set user resource, route
+    // TODO: users-next: this belongs in the web layer only
     userOut.avatarUrl = [(options.path ? options.path : ""), "api", "users", user._id, "avatar"].join("/");
   }
 
   if (user.icon && user.icon.relativePath) {
-    // TODO, don't really like this, need a better way to set user resource, route
+    // TODO: users-next: this belongs in the web layer only
     userOut.iconUrl = [(options.path ? options.path : ""), "api", "users", user._id, "icon"].join("/");
   }
 
   return userOut;
-};
+}
+
+export function UserModel(conn: mongoose.Connection): UserModel {
+  return conn.model(UserModelName, UserSchema, 'users')
+}
 
+/**
+ * Return the string value of the MongoDB ObjectID or Mongoose document's ID.
+ */
 const idString = (x: mongoose.Document | mongoose.Types.ObjectId): string => {
   const id: mongoose.Types.ObjectId = x instanceof mongoose.Document ? x._id : x
   return id.toHexString()
 }
 
-export class MongooseUserRepository extends BaseMongooseRepository<UserDocument, UserModel, User> implements UserRepository {
-
-  constructor(model: mongoose.Model<UserDocument>) {
-    super(model, {
-      docToEntity: doc => {
-        const json = doc.toJSON()
-        return {
-          ...json,
-          id: doc._id.toHexString(),
-          roleId: idString(doc.roleId),
-          authenticationId: idString(doc.authenticationId)
-        }
-      }
-    })
+type EntityTypeForDocType<DocType> = DocType extends UserExpandedDocument | UserExpandedModelInstance ? UserExpanded : User
+
+function entityForDoc<DocType extends UserDocument | UserModelInstance | UserExpandedDocument | UserExpandedModelInstance>(from: DocType): EntityTypeForDocType<DocType> {
+  const doc = from instanceof mongoose.Document ? from.toObject() : from
+  const entity: any = {
+    ...doc,
+    id: from._id.toHexString(),
+  }
+  if (doc.roleId instanceof ObjectId) {
+    entity.roleId = idString(doc.roleId)
+    return entity
+  }
+  const { _id: docRoleId, ...partialRole } = doc.roleId
+  const roleId = docRoleId.toHexString()
+  entity.roleId = roleId
+  entity.role =  { ...partialRole, id: roleId }
+  return entity
+}
+
+export class MongooseUserRepository implements UserRepository {
+
+  private baseRepo: BaseMongooseRepository<UserDocument, UserModel, User>
+
+  constructor(private model: UserModel) {
+    this.baseRepo = new BaseMongooseRepository<UserDocument, UserModel, User>(model, { docToEntity: entityForDoc })
   }
 
-  async create(): Promise<User> {
-    throw new Error('method not allowed')
+  findById(id: string): Promise<UserExpanded | null> {
+    return this.model.findById<UserExpandedDocument>(id, null, { populate: 'roleId', lean: true }).then(x => x ? entityForDoc(x) : null)
+  }
+
+  findAllByIds(ids: string[]): Promise<{ [id: string]: User | null }> {
+    return this.baseRepo.findAllByIds(ids)
+  }
+
+  async create(attrs: Omit<User, 'id' | 'icon' | 'avatar'>): Promise<UserExpanded | UserRepositoryError> {
+    try {
+      const userOrm = await this.model.create(attrs)
+      const userExpandedDoc = await userOrm.populate<Pick<UserExpandedModelInstance, 'roleId'>>('roleId') as UserExpandedModelInstance
+      return entityForDoc(userExpandedDoc)
+    }
+    catch (err) {
+      // TODO: duplicate username error
+      return new UserRepositoryError(UserRepositoryErrorCode.StorageError, String(err))
+    }
   }
 
   async update(attrs: Partial<User> & { id: UserId }): Promise<User | null> {
     throw new Error('method not allowed')
   }
 
-  async removeById(id: any): Promise<User | null> {
+  async removeById(id: UserId): Promise<User | null> {
     throw new Error('method not allowed')
   }
 
+  async findByUsername(username: string): Promise<UserExpanded | null> {
+    const doc = await this.model.findOne<UserExpandedDocument>({ username }, null, { populate: 'roleId', lean: true })
+    if (doc) {
+      return entityForDoc(doc)
+    }
+    return null
+  }
+
   async find<T = User>(which: UserFindParameters, mapping?: (x: User) => T): Promise<PageOf<T>> {
-    const { nameOrContactTerm, active, enabled } = which || {}
+    const { nameOrContactTerm, active, enabled } = (which || {})
     const searchRegex = new RegExp(_.escapeRegExp(nameOrContactTerm), 'i')
-
     const params = nameOrContactTerm ? {
       $or: [
         { username: searchRegex },
@@ -273,27 +293,37 @@ export class MongooseUserRepository extends BaseMongooseRepository<UserDocument,
         { 'phones.number': searchRegex }
       ]
     } : {} as any
-
-    if (typeof which?.active === 'boolean') {
+    if (typeof active === 'boolean') {
       params.active = which.active
-
     }
-    if (typeof which?.enabled === 'boolean') {
+    if (typeof enabled === 'boolean') {
       params.enabled = which.enabled
     }
-
     const baseQuery = this.model.find(params).sort('displayName _id')
     const counted = await pageQuery(baseQuery, which)
     const users: T[] = []
     if (!mapping) {
-      mapping = (x: User) => (x as any as T)
+      mapping = (x: User): T => (x as any as T)
     }
     for await (const userDoc of counted.query.cursor()) {
-      users.push(mapping(this.entityForDocument(userDoc)))
+      users.push(mapping(entityForDoc(userDoc)))
     }
+    return pageOf(users, which, counted.totalCount)
+  }
 
-    const finalResult = pageOf(users, which, counted.totalCount);
-    return finalResult;
+  saveMapIcon(userId: UserId, icon: UserIcon, content: NodeJS.ReadableStream | Buffer): Promise<User | UserRepositoryError> {
+    throw new Error('Method not implemented.')
+  }
+
+  saveAvatar(userId: UserId, avatar: Avatar, content: NodeJS.ReadableStream | Buffer): Promise<User | UserRepositoryError> {
+    throw new Error('Method not implemented.')
+  }
 
+  deleteMapIcon(userId: UserId): Promise<User | UserRepositoryError> {
+    throw new Error('Method not implemented.')
   }
-}
\ No newline at end of file
+
+  deleteAvatar(userId: UserId): Promise<User | UserRepositoryError> {
+    throw new Error('Method not implemented.')
+  }
+}
diff --git a/service/src/entities/users/entities.users.ts b/service/src/entities/users/entities.users.ts
index 74675ec6d..1da61041b 100644
--- a/service/src/entities/users/entities.users.ts
+++ b/service/src/entities/users/entities.users.ts
@@ -34,8 +34,9 @@ export interface User {
   recentEventIds: MageEventId[]
 }
 
-export type UserExpanded = Omit<User, 'roleId'>
-  & { role: Role }
+export interface UserExpanded extends User {
+  role: Role
+}
 
 export interface Phone {
   type: string,

From 60a97a045c33b3aeb597502f40379e2638f00611 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Fri, 29 Nov 2024 15:09:28 -0700
Subject: [PATCH 169/183] refactor(service): users/auth: fix admission token
 reference on ingress response

---
 .../ingress.adapters.controllers.web.ts       | 44 +++++++------------
 1 file changed, 17 insertions(+), 27 deletions(-)

diff --git a/service/src/ingress/ingress.adapters.controllers.web.ts b/service/src/ingress/ingress.adapters.controllers.web.ts
index bcdf9ae37..40650a037 100644
--- a/service/src/ingress/ingress.adapters.controllers.web.ts
+++ b/service/src/ingress/ingress.adapters.controllers.web.ts
@@ -4,12 +4,10 @@ import { Authenticator } from 'passport'
 import { Strategy as BearerStrategy } from 'passport-http-bearer'
 import { defaultHashUtil } from '../utilities/password-hashing'
 import { JWTService, Payload, TokenVerificationError, VerificationErrorReason, TokenAssertion } from './verification'
-import { invalidInput, InvalidInputError, MageError, permissionDenied } from '../app.api/app.api.errors'
-import { IdentityProvider, IdentityProviderRepository } from './ingress.entities'
+import { invalidInput, InvalidInputError, MageError } from '../app.api/app.api.errors'
+import { IdentityProvider } from './ingress.entities'
 import { AdmitFromIdentityProviderOperation, EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
 import { IngressProtocolWebBinding, IngressResponseType } from './ingress.protocol.bindings'
-import { createWebBinding as LocalBinding } from './ingress.protocol.local'
-import { createWebBinding as OAuthBinding } from './ingress.protocol.oauth'
 
 declare module 'express-serve-static-core' {
   interface Request {
@@ -43,37 +41,29 @@ export type IngressUseCases = {
   admitFromIdentityProvider: AdmitFromIdentityProviderOperation
 }
 
+export type IngressProtocolWebBindingCache = {
+  idpWebBindingForIdpName(idpName: string): Promise<{ idp: IdentityProvider, idpBinding: IngressProtocolWebBinding } | null>
+}
+
 export type IngressRoutes = {
   localEnrollment: express.Router
   idpAdmission: express.Router
 }
 
-export function CreateIngressRoutes(ingressApp: IngressUseCases, idpRepo: IdentityProviderRepository, tokenService: JWTService, passport: Authenticator): IngressRoutes {
-
-  const idpBindings = new Map<string, IngressProtocolWebBinding>()
-  async function bindingFor(idpName: string): Promise<IngressProtocolWebBinding | null> {
-    const idp = await idpRepo.findIdpByName(idpName)
-    if (!idp) {
-      return null
-    }
-    if (idp.protocol === 'local') {
-      return LocalBinding(passport, )
-    }
-    throw new Error('unimplemented')
-  }
+export function CreateIngressRoutes(ingressApp: IngressUseCases, idpCache: IngressProtocolWebBindingCache, tokenService: JWTService, passport: Authenticator): IngressRoutes {
 
   const routeToIdp = express.Router()
     .all('/',
       ((req, res, next) => {
-        const idpService = req.ingress?.idpBinding
-        if (!idpService) {
+        const idpBinding = req.ingress?.idpBinding
+        if (!idpBinding) {
           return next(new Error(`no identity provider for ingress request: ${req.method} ${req.originalUrl}`))
         }
         if (req.path.endsWith('/signin')) {
           const userAgentType: UserAgentType = req.params.state === 'mobile' ? UserAgentType.MobileApp : UserAgentType.WebApp
-          return idpService.beginIngressFlow(req, res, next, userAgentType)
+          return idpBinding.beginIngressFlow(req, res, next, userAgentType)
         }
-        idpService.handleIngressFlowRequest(req, res, next)
+        idpBinding.handleIngressFlowRequest(req, res, next)
       }) as express.RequestHandler,
       (async (err, req, res, next) => {
         if (err) {
@@ -97,7 +87,7 @@ export function CreateIngressRoutes(ingressApp: IngressUseCases, idpRepo: Identi
         }
         if (idpAdmission.flowState === UserAgentType.MobileApp) {
           if (mageAccount.active && mageAccount.enabled) {
-            return res.redirect(`mage://app/authentication?token=${req.token}`)
+            return res.redirect(`mage://app/authentication?token=${admissionToken}`)
           }
           else {
             return res.redirect(`mage://app/invalid_account?active=${mageAccount.active}&enabled=${mageAccount.enabled}`)
@@ -115,15 +105,15 @@ export function CreateIngressRoutes(ingressApp: IngressUseCases, idpRepo: Identi
   idpAdmission.use('/:identityProviderName',
     async (req, res, next) => {
       const idpName = req.params.identityProviderName
-      const idp = await idpRepo.findIdpByName(idpName)
-      const idpBinding = await bindingFor(idpName)
-      if (idpBinding && idp) {
-        req.ingress = { idpBinding, idp }
+      const idpBindingEntry = await idpCache.idpWebBindingForIdpName(idpName)
+      if (idpBindingEntry) {
+        const { idp, idpBinding } = idpBindingEntry
+        req.ingress = { idp, idpBinding }
         return next()
       }
       res.status(404).send(`${idpName} not found`)
     },
-    // use a sub-router so express implicitly strips the base url /auth/:identityProviderName before routing idp handler
+    // use a sub-router so express implicitly strips the base url /auth/:identityProviderName before routing to idp handler
     routeToIdp
   )
 

From ac166f95a5312f6e960e8fdaf41dcd022610d8a8 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 30 Nov 2024 14:29:28 -0700
Subject: [PATCH 170/183] refactor(service): users/auth: renames for
 consistency

---
 service/src/ingress/sessions.adapters.db.mongoose.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/service/src/ingress/sessions.adapters.db.mongoose.ts b/service/src/ingress/sessions.adapters.db.mongoose.ts
index 541641153..652ca352d 100644
--- a/service/src/ingress/sessions.adapters.db.mongoose.ts
+++ b/service/src/ingress/sessions.adapters.db.mongoose.ts
@@ -1,6 +1,6 @@
 import crypto from 'crypto'
 import mongoose, { Schema } from 'mongoose'
-import { UserDocumentExpanded } from '../adapters/users/adapters.users.db.mongoose'
+import { UserExpandedDocument } from '../adapters/users/adapters.users.db.mongoose'
 import { UserId } from '../entities/users/entities.users'
 import { Session, SessionRepository } from './ingress.entities'
 
@@ -10,8 +10,8 @@ export interface SessionDocument {
   userId: mongoose.Types.ObjectId
   deviceId?: mongoose.Types.ObjectId | undefined
 }
-export type SessionDocumentExpanded = SessionDocument & {
-  userId: UserDocumentExpanded
+export type SessionExpandedDocument = SessionDocument & {
+  userId: UserExpandedDocument
 }
 export type SessionModel = mongoose.Model<SessionDocument>
 

From 4236c0e8e59e549c6de952caee4bfc1f043092df Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 30 Nov 2024 16:55:24 -0700
Subject: [PATCH 171/183] refactor(service): users/auth: reconcile passport
 types in ingress protocols

---
 service/src/ingress/ingress.protocol.oauth.ts | 4 ++--
 service/src/ingress/ingress.protocol.saml.ts  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/service/src/ingress/ingress.protocol.oauth.ts b/service/src/ingress/ingress.protocol.oauth.ts
index 992c9570c..effe790b1 100644
--- a/service/src/ingress/ingress.protocol.oauth.ts
+++ b/service/src/ingress/ingress.protocol.oauth.ts
@@ -2,7 +2,7 @@ import express from 'express'
 import { InternalOAuthError, Strategy as OAuth2Strategy, StrategyOptions as OAuth2Options, VerifyCallback, VerifyFunction } from 'passport-oauth2'
 import base64 from 'base-64'
 import { IdentityProvider, IdentityProviderUser } from './ingress.entities'
-import { Authenticator } from 'passport'
+import passport from 'passport'
 import { IngressProtocolWebBinding, IngressResponseType } from './ingress.protocol.bindings'
 
 export type OAuth2ProtocolSettings =
@@ -88,7 +88,7 @@ type OAuth2Info = { state: string }
  * The `baseUrl` parameter is the URL at which Mage will mount the returned `express.Router`, including any
  * distinguishing component of the given `IdentityProvider`, without a trailing slash, e.g. `/auth/example-idp`.
  */
-export function createOAuthProtocolWebBinding(idp: IdentityProvider, passport: Authenticator, baseUrl: string): IngressProtocolWebBinding {
+export function createOAuthProtocolWebBinding(idp: IdentityProvider, passport: passport.Authenticator, baseUrl: string): IngressProtocolWebBinding {
   const settings = applyDefaultProtocolSettings(idp)
   const profileURL = settings.profileURL
   const customHeaders = settings.headers?.basic ? {
diff --git a/service/src/ingress/ingress.protocol.saml.ts b/service/src/ingress/ingress.protocol.saml.ts
index c4c48c0ab..b580ffe15 100644
--- a/service/src/ingress/ingress.protocol.saml.ts
+++ b/service/src/ingress/ingress.protocol.saml.ts
@@ -1,5 +1,5 @@
 import express from 'express'
-import { Authenticator } from 'passport'
+import passport from 'passport'
 import { SamlConfig, Strategy as SamlStrategy, VerifyWithRequest } from '@node-saml/passport-saml'
 import { IdentityProvider, IdentityProviderUser } from './ingress.entities'
 import { IngressProtocolWebBinding, IngressResponseType } from './ingress.protocol.bindings'
@@ -64,7 +64,7 @@ function applyDefaultProtocolSettings(idp: IdentityProvider): SamlProtocolSettin
   return settings
 }
 
-export function createSamlProtocolWebBinding(idp: IdentityProvider, passport: Authenticator, baseUrlPath: string): IngressProtocolWebBinding {
+export function createSamlProtocolWebBinding(idp: IdentityProvider, passport: passport.Authenticator, baseUrlPath: string): IngressProtocolWebBinding {
   const { profile: profileKeys, ...settings } = applyDefaultProtocolSettings(idp)
   // TODO: this will need the the saml callback override change
   settings.path = `${baseUrlPath}/callback`

From a41d0fb0f9e6a15ebcb913bf15596a35f4815b68 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 30 Nov 2024 16:56:26 -0700
Subject: [PATCH 172/183] refactor(service): users/auth: moving initialization
 from ingress index to ingress main module

---
 service/src/ingress/index.ts        | 110 +---------------------------
 service/src/ingress/ingress.main.ts |  89 +++++++++++++++++++++-
 2 files changed, 91 insertions(+), 108 deletions(-)

diff --git a/service/src/ingress/index.ts b/service/src/ingress/index.ts
index abec4966d..b355346ac 100644
--- a/service/src/ingress/index.ts
+++ b/service/src/ingress/index.ts
@@ -6,7 +6,7 @@ import provision, { ProvisionStatic } from '../provision'
 import { User, UserRepository } from '../entities/users/entities.users'
 import bearer from 'passport-http-bearer'
 import { UserDocument } from '../adapters/users/adapters.users.db.mongoose'
-import { SessionRepository } from './entities.authentication'
+import { SessionRepository } from './ingress.entities'
 const api = require('../api/')
 const config = require('../config.js')
 const log = require('../logger')
@@ -16,90 +16,10 @@ const AuthenticationConfiguration = require('../models/authenticationconfigurati
 const SecurePropertyAppender = require('../security/utilities/secure-property-appender');
 
 
-
-export async function initializeAuthenticationStack(
-  userRepo: UserRepository,
-  sessionRepo: SessionRepository,
-  verificationService: JWTService,
-  provisioning: provision.ProvisionStatic,
-  passport: passport.Authenticator,
-): Promise<express.Router> {
-  passport.serializeUser((user, done) => done(null, user.id))
-  passport.deserializeUser(async (id, done) => {
-    try {
-      const user = await userRepo.findById(String(id))
-      done(null, user)
-    }
-    catch (err) {
-      done(err)
-    }
-  })
-  registerAuthenticatedBearerTokenHandling(passport, sessionRepo, userRepo)
-  registerIdpAuthenticationVerification(passport, verificationService, userRepo)
-  const routes = express.Router()
-  registerTokenGenerationEndpointWithDeviceVerification(routes, passport)
-}
-
-const VerifyIdpAuthenticationToken = 'verifyIdpAuthenticationToken'
-
-function registerAuthenticatedBearerTokenHandling(passport: passport.Authenticator, sessionRepo: SessionRepository, userRepo: UserRepository): passport.Authenticator {
-  return passport.use(
-    /*
-    This is the default bearer token authentication, registered to the passport instance under the default `bearer`
-    name.
-    */
-    new bearer.Strategy(
-      { passReqToCallback: true },
-      async function (req: express.Request, token: string, done: (err: Error | null, user?: User, access?: bearer.IVerifyOptions) => any) {
-        try {
-          const session = await sessionRepo.findSessionByToken(token)
-          if (!session) {
-            console.warn('no session for token', token, req.method, req.url)
-            return done(null)
-          }
-          const user = await userRepo.findById(session.user)
-          if (!user) {
-            console.warn('no user for token', token, 'user id', session.user, req.method, req.url)
-            return done(null)
-          }
-          req.token = session.token
-          if (session.device) {
-            req.provisionedDeviceId = session.device
-          }
-          return done(null, user, { scope: 'all' });
-        }
-        catch (err) {
-          return done(err as Error)
-        }
-      }
-    )
-  )
-}
-
 /**
- * Register a `BearerStrategy` that expects a JWT in the `Authorization` header that contains the
- * {@link TokenAssertion.Authorized} claim.  The claim indicates the subject has authenticated with an IDP and can
- * continue the sign-in process.  Decode and verify the JWT signature, retrieve the `User` for the JWT subject, and set
- * `Request.user`.
+ * Register the route to generate an API access token, the final step in the ingress process after enrollment,
+ * authentication.  This step includes provisioning a device based on the configured policy.
  */
-function registerIdpAuthenticationVerification(passport: passport.Authenticator, verificationService: JWTService, userRepo: UserRepository): passport.Authenticator {
-  passport.use(VerifyIdpAuthenticationToken, new bearer.Strategy(async function(token, done: (error: any, user?: User) => any) {
-    try {
-      const expectation: Payload = { assertion: TokenAssertion.Authorized, subject: null, expiration: null }
-      const payload = await verificationService.verifyToken(token, expectation)
-      const user = payload.subject ? await userRepo.findById(payload.subject) : null
-      if (user) {
-        return done(null, user)
-      }
-      done(new Error(`user id ${payload.subject} not found for transient token ${String(payload)}`))
-    }
-    catch (err) {
-      done(err)
-    }
-  }))
-  return passport
-}
-
 function registerDeviceVerificationAndTokenGenerationEndpoint(routes: express.Router, passport: passport.Authenticator, deviceProvisioning: ProvisionStatic, sessionRepo: SessionRepository) {
   routes.post('/auth/token',
     passport.authenticate(VerifyIdpAuthenticationToken),
@@ -158,31 +78,9 @@ export class AuthenticationInitializer {
       });
     });
 
-    passport.use(
-      new bearer.Strategy(
-        { passReqToCallback: true },
-        async function (req: express.Request, token: string, done: (err: Error | null, user?: UserDocument, access?: bearer.IVerifyOptions) => any) {
-          try {
-            const session = await lookupSession(token)
-            if (!session || !session.user) {
-              return done(null)
-            }
-            req.token = session.token;
-            if (session.deviceId) {
-              req.provisionedDeviceId = session.deviceId.toHexString();
-            }
-            return done(null, session.user, { scope: 'all' });
-          }
-          catch (err) {
-            return done(err as Error)
-          }
-        }
-      )
-    )
-
     passport.use('authorization', new BearerStrategy(function (token, done) {
       const expectation = {
-        assertion: TokenAssertion.Authorized
+        assertion: TokenAssertion.Authenticated
       };
 
       AuthenticationInitializer.tokenService.verifyToken(token, expectation)
diff --git a/service/src/ingress/ingress.main.ts b/service/src/ingress/ingress.main.ts
index fec508993..e23090e95 100644
--- a/service/src/ingress/ingress.main.ts
+++ b/service/src/ingress/ingress.main.ts
@@ -1,6 +1,9 @@
+import express from 'express'
 import passport from 'passport'
+import bearer from 'passport-http-bearer'
+import { UserRepository } from '../entities/users/entities.users'
 import { IngressProtocolWebBindingCache } from './ingress.adapters.controllers.web'
-import { IdentityProvider, IdentityProviderRepository } from './ingress.entities'
+import { IdentityProvider, IdentityProviderRepository, SessionRepository } from './ingress.entities'
 import { IngressProtocolWebBinding } from './ingress.protocol.bindings'
 import { createLdapProtocolWebBinding } from './ingress.protocol.ldap'
 import { createLocalProtocolWebBinding } from './ingress.protocol.local'
@@ -8,6 +11,7 @@ import { createOAuthProtocolWebBinding } from './ingress.protocol.oauth'
 import { createOIDCProtocolWebBinding } from './ingress.protocol.oidc'
 import { createSamlProtocolWebBinding } from './ingress.protocol.saml'
 import { MageLocalIdentityProviderService } from './local-idp.services.api'
+import { JWTService } from './verification'
 
 export function createIdpCache(idpRepo: IdentityProviderRepository, localIdp: MageLocalIdentityProviderService): IngressProtocolWebBindingCache {
   const bindingsByIdpName = new Map<string, { idp: IdentityProvider, idpBinding: IngressProtocolWebBinding }>()
@@ -35,11 +39,92 @@ export function createIdpCache(idpRepo: IdentityProviderRepository, localIdp: Ma
 type BindingServices = {
   passport: passport.Authenticator
   localIdp: MageLocalIdentityProviderService
+  baseUrl: string
 }
 
-function createWebBinding(idp: IdentityProvider, services: BindingServices) {
+function createWebBinding(idp: IdentityProvider, services: BindingServices): IngressProtocolWebBinding {
   if (idp.protocol === 'local') {
     return createLocalProtocolWebBinding(services.passport, services.localIdp)
   }
+  if (idp.protocol === 'ldap') {
+    return createLdapProtocolWebBinding(idp, services.passport)
+  }
+  if (idp.protocol === 'oauth') {
+    return createOAuthProtocolWebBinding(idp, services.passport, services.baseUrl)
+  }
+  if (idp.protocol === 'oidc') {
+    return createOIDCProtocolWebBinding(idp, services.passport, services.baseUrl)
+  }
+  if (idp.protocol === 'saml') {
+    return createSamlProtocolWebBinding(idp, services.passport, services.baseUrl)
+  }
   throw new Error(`cannot create ingress web binding for idp:\n${JSON.stringify(idp, null, 2)}`)
+}
+
+export async function initializeIngress(
+  userRepo: UserRepository,
+  sessionRepo: SessionRepository,
+  verificationService: JWTService,
+  provisioning: provision.ProvisionStatic,
+  passport: passport.Authenticator,
+): Promise<express.Router> {
+  // TODO: users-next: these serialization functions are probably no longer necessary
+  passport.serializeUser((user, done) => done(null, user.id))
+  passport.deserializeUser(async (id, done) => {
+    try {
+      const user = await userRepo.findById(String(id))
+      done(null, user)
+    }
+    catch (err) {
+      done(err)
+    }
+  })
+  const routes = express.Router()
+  registerAuthenticatedBearerTokenHandling(passport, sessionRepo, userRepo)
+  return routes
+}
+
+/**
+ * This is the default bearer token authentication, registered to the passport instance under the default `bearer`
+ * name.  Apply session token authentication to routes using Passport's middelware factory as follows.
+ * ```
+ * expressRoutes.route('/protected')
+ *   .use(passport.authenticate('bearer'))
+ *   .get((req, res, next) => { ... })
+ * ```
+ */
+function registerAuthenticatedBearerTokenHandling(passport: passport.Authenticator, sessionRepo: SessionRepository, userRepo: UserRepository): passport.Authenticator {
+  return passport.use(
+    new bearer.Strategy(
+      { passReqToCallback: true },
+      async function (req: express.Request, token: string, done: (err: Error | null, user?: Express.User, access?: bearer.IVerifyOptions) => any) {
+        try {
+          const session = await sessionRepo.readSessionByToken(token)
+          if (!session) {
+            console.warn('no session for token', token, req.method, req.url)
+            return done(null)
+          }
+          const user = await userRepo.findById(session.user)
+          if (!user) {
+            console.warn('no user for token', token, 'user id', session.user, req.method, req.url)
+            return done(null)
+          }
+          req.token = session.token
+          if (session.device) {
+            req.provisionedDeviceId = session.device
+          }
+          const webUser: Express.User = {
+            admitted: {
+              account: user,
+              session
+            }
+          }
+          return done(null, webUser, { scope: 'all' });
+        }
+        catch (err) {
+          return done(err as Error)
+        }
+      }
+    )
+  )
 }
\ No newline at end of file

From ac84fefc5ec50d3ba693ef43567fd2f8ccc64711 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 30 Nov 2024 22:22:42 -0700
Subject: [PATCH 173/183] refactor(service): users/auth: ingress readme

---
 service/src/ingress/readme.md | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 service/src/ingress/readme.md

diff --git a/service/src/ingress/readme.md b/service/src/ingress/readme.md
new file mode 100644
index 000000000..66ce239f6
--- /dev/null
+++ b/service/src/ingress/readme.md
@@ -0,0 +1,34 @@
+# Mage Ingress
+
+Ingress is the process Mage users to enroll, authenticate, and admit users to establish sessions.
+
+## Concepts
+
+**User:** The person that requires access to the Mage service and resources
+
+**Account:** The persistent record in Mage that represents and tracks a user
+
+**Enrollment:** Creating a new Mage account a user will use to access Mage resources
+
+**Authentication:** Authentication is the verification of a user's credentials to prove
+
+**Identity Provider:** An identity provider is a service with the sole responsibility to authenticate users using a
+specific protocol, from the perspective of Mage.  Examples include Google OAuth, Google OIDC, Meta, a corporate
+enterprise LDAP server, etc.
+
+**Ingress Protocol:** An ingress protocol defines the sequence of interactions between a user, an identity provider,
+and the Mage service that delegates user authentication to the identity provider in order to allow access
+to Mage resources.  A typical example is a service which allows a user to authenticate with a Google account through
+a series of HTTP requests and redirects.
+
+## Flow
+
+1. User chooses the identity provider.
+1. Web app requests sign-in to identity provider.
+1. Server directs sign-in request to IDP.
+1. IDP returns result to server.
+1. Server maps IDP account to Mage user account, creating account if necessary, to enroll the user.
+   1. If account inactive or disabled, skip JWT, return { user, token: null }
+   1. If account is active and enabled, generate authenticated JWT, return { user, token }
+1. Client sends JWT and device information to server to request API session token.
+1. Server verifies JWT, applies device enrollment policy, creates a session, and returns the session token to the client.
\ No newline at end of file

From 0f0d5ccf31dc8e9b1e74cbec4e5005a65da7a0ec Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 30 Nov 2024 23:14:08 -0700
Subject: [PATCH 174/183] refactor(service): users/auth: migrations

---
 .../src/migrations/032-rename-authconfigs.ts  | 31 +++++++++++++++++
 .../migrations/033-user-idp-relationship.ts   | 20 +++++++++++
 .../src/migrations/034-ingress-refactor.ts    | 33 +++++++++++++++++++
 3 files changed, 84 insertions(+)
 create mode 100644 service/src/migrations/032-rename-authconfigs.ts
 create mode 100644 service/src/migrations/033-user-idp-relationship.ts
 create mode 100644 service/src/migrations/034-ingress-refactor.ts

diff --git a/service/src/migrations/032-rename-authconfigs.ts b/service/src/migrations/032-rename-authconfigs.ts
new file mode 100644
index 000000000..1aea3a2f1
--- /dev/null
+++ b/service/src/migrations/032-rename-authconfigs.ts
@@ -0,0 +1,31 @@
+// import { Migration } from '@ngageoint/mongodb-migrations'
+
+// const migration: Migration = {
+
+//   id: 'rename-authconfigs',
+
+//   up: async function(done) {
+//     this.log('rename authenticationconfigurations to identity_providers')
+//     try {
+//       await this.db.renameCollection('authenticationconfigurations', 'identity_providers')
+//       done(null)
+//     }
+//     catch (err) {
+//       done(err)
+//     }
+//     done(null)
+//   },
+
+//   down: async function(done) {
+//     this.log('rename identity_providers to authenticationconfigurations')
+//     try {
+//       await this.db.renameCollection('identity_providers', 'authenticationconfigurations')
+//       done(null)
+//     }
+//     catch (err) {
+//       done(err)
+//     }
+//   }
+// }
+
+// export = migration
\ No newline at end of file
diff --git a/service/src/migrations/033-user-idp-relationship.ts b/service/src/migrations/033-user-idp-relationship.ts
new file mode 100644
index 000000000..6f31944e9
--- /dev/null
+++ b/service/src/migrations/033-user-idp-relationship.ts
@@ -0,0 +1,20 @@
+import { Migration } from '@ngageoint/mongodb-migrations'
+
+/**
+ * Invert the relationship between users and identity providers by migrating `users.authenticationId` to
+ * `user_ingress_bindings.userId`.
+ */
+const migration: Migration = {
+
+  id: 'user-idp-relationship',
+
+  up: async function(done) {
+    done(new Error('unimplemented'))
+  },
+
+  down: async function(done) {
+    done(new Error('unimplemented'))
+  }
+}
+
+export = migration
\ No newline at end of file
diff --git a/service/src/migrations/034-ingress-refactor.ts b/service/src/migrations/034-ingress-refactor.ts
new file mode 100644
index 000000000..b3cf8b018
--- /dev/null
+++ b/service/src/migrations/034-ingress-refactor.ts
@@ -0,0 +1,33 @@
+// import { Migration } from '@ngageoint/mongodb-migrations'
+
+// /**
+//  * **TODO**
+//  * * rename `type` to `protocol`
+//  * * rename `settings` to `protocolSettings`
+//  * * migrate common policy from `authenticationconfigurations.settings` to `identity_providers` `userEnrollmentPolicy` and `deviceEnrollmentPolicy`
+//  *   * usersReqAdmin, devicesReqAdmin, newUserTeams, newUserEvents
+//  * * add `assignRole` with default USER_ROLE ID to `userEnrollmentPolicy`
+//  * * move `authentications` to `User.ingressAccounts` array `{ identityProviderId: ObjectId, enabled: boolean, accountSettings?: Mixed }`
+//  * * move `authentications` with local idp to `local_idp_accounts`
+//  */
+// const migration: Migration = {
+
+//   id: 'ingress-refactor',
+
+//   async up(done) {
+//     const { db, log } = this
+//     const idps = db.collection('identity_providers')
+//     const localIdps = await idps.find({ type: 'local' }).toArray()
+//     if (localIdps.length !== 1) {
+//       return done(new Error(`unexpected `))
+//     }
+//     const localIdp = localIdps[0]
+//     const localIdpId = localIdp._id
+//   },
+
+//   async down(done) {
+
+//   }
+// }
+
+// export = migration
\ No newline at end of file

From 764f918cec8193456fa27444476b28d09ed379b0 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 30 Nov 2024 23:21:02 -0700
Subject: [PATCH 175/183] refactor(service): users/auth: renamed session
 repository methods

---
 .../src/adapters/devices/adapters.devices.controllers.web.ts  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/service/src/adapters/devices/adapters.devices.controllers.web.ts b/service/src/adapters/devices/adapters.devices.controllers.web.ts
index 69353bb1a..b14d6cba8 100644
--- a/service/src/adapters/devices/adapters.devices.controllers.web.ts
+++ b/service/src/adapters/devices/adapters.devices.controllers.web.ts
@@ -58,7 +58,7 @@ export function DeviceRoutes(deviceRepo: DeviceRepository, userRepo: UserReposit
         try {
           if (update.registered === false) {
             console.info(`update device ${idInPath} to unregistered`)
-            const sessionsRemovedCount = await sessionRepo.removeSessionsForDevice(idInPath)
+            const sessionsRemovedCount = await sessionRepo.deleteSessionsForDevice(idInPath)
             console.info(`removed ${sessionsRemovedCount} session(s) for device ${idInPath}`)
           }
           const updated = await deviceRepo.update({ ...update, id: idInPath })
@@ -79,7 +79,7 @@ export function DeviceRoutes(deviceRepo: DeviceRepository, userRepo: UserReposit
           const idInPath = req.params.id
           console.info(`delete device`, idInPath)
           const deleted = await deviceRepo.removeById(idInPath)
-          const removedSessionsCount = sessionRepo.removeSessionsForDevice(idInPath)
+          const removedSessionsCount = sessionRepo.deleteSessionsForDevice(idInPath)
           console.info(`removed ${removedSessionsCount} session(s) for device ${idInPath}`)
           // TODO: the old observation model had a middleware that removed the device id from created observations,
           // but do we really care that much

From 08a1dc718b9336759c66ee05a09436feb1915f77 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 30 Nov 2024 23:21:20 -0700
Subject: [PATCH 176/183] refactor(service): users/auth: mongoose readme

---
 docs/development/mongoose.md | 38 ++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 docs/development/mongoose.md

diff --git a/docs/development/mongoose.md b/docs/development/mongoose.md
new file mode 100644
index 000000000..57dd5a28b
--- /dev/null
+++ b/docs/development/mongoose.md
@@ -0,0 +1,38 @@
+# Mongoose Development Guidelines and Patterns
+
+## Types
+* DocType vs. Entity
+* Entity with JsonObject mapped to DocType causes TS error
+  _Type instantiation is excessively deep and possibly infinite.ts(2589)_
+  because of JsonObject recursive type definition
+```
+export type FeedServiceDocument = Omit<FeedService, 'id' | 'serviceType'> & {
+  _id: mongoose.Types.ObjectId
+  serviceType: mongoose.Types.ObjectId
+  // config: any
+}
+export type FeedServiceModel = Model<FeedServiceDocument>
+export const FeedServiceSchema = new mongoose.Schema<FeedServiceDocument, FeedServiceModel>(
+  {
+    serviceType: { type: mongoose.SchemaTypes.ObjectId, required: true, ref: FeedsModels.FeedServiceTypeIdentity },
+    title: { type: String, required: true },
+    summary: { type: String, required: false },
+    config: { type: Object, required: false },
+  },
+  {
+    toJSON: {
+      getters: true,
+      versionKey: false,
+      transform: (doc: FeedServiceDocument, json: any & FeedService): void => {
+        delete json._id
+        json.serviceType = doc.serviceType.toHexString()
+      }
+    }
+  })
+```
+
+```
+var o: ObservationDocument = null
+var l: mongoose.LeanDocument<ObservationDocument> = null
+o = l
+```
\ No newline at end of file

From ae54610a86dc0f07c4beb025f31e1db2aab1715d Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 30 Nov 2024 23:24:33 -0700
Subject: [PATCH 177/183] refactor(service): users/auth: role database type
 changes

---
 service/src/models/role.d.ts | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/service/src/models/role.d.ts b/service/src/models/role.d.ts
index f6784fa6a..b5f6a4bf7 100644
--- a/service/src/models/role.d.ts
+++ b/service/src/models/role.d.ts
@@ -1,20 +1,22 @@
-
+import mongoose from 'mongoose'
 import { AnyPermission } from '../entities/authorization/entities.permissions'
 
 type Callback<R> = (err: any, result?: R) => any
 
 export declare interface RoleDocument {
-  id: string
+  _id: mongoose.Types.ObjectId
   name: string
   description?: string
   permissions: AnyPermission[]
 }
 
+export type RoleModelInstance = mongoose.HydratedDocument<RoleDocument>
+
 export declare type RoleJson = Omit<RoleDocument, '_id'>
 
-export declare function getRoleById(id: string, callback: Callback<RoleDocument | null>): void
-export declare function getRole(name: string, callback: Callback<RoleDocument | null>): void
-export declare function getRoles(callback: Callback<RoleDocument[]>): void
-export declare function createRole(role: RoleDocument, callback: Callback<RoleDocument>): void
-export declare function updateRole(id: string, update: Partial<RoleDocument>, callback: Callback<RoleDocument>): void
-export declare function deleteRole(role: RoleDocument, callback: Callback<RoleDocument>): void
+export declare function getRoleById(id: string, callback: Callback<RoleModelInstance | null>): void
+export declare function getRole(name: string, callback: Callback<RoleModelInstance | null>): void
+export declare function getRoles(callback: Callback<RoleModelInstance[]>): void
+export declare function createRole(role: Omit<RoleDocument, '_id'>, callback: Callback<RoleModelInstance>): void
+export declare function updateRole(id: string, update: Partial<RoleDocument>, callback: Callback<RoleModelInstance>): void
+export declare function deleteRole(role: RoleModelInstance, callback: Callback<RoleModelInstance>): void

From 12e97761ec42276c3576b01a2bd415c29e0fa2dc Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 30 Nov 2024 23:26:14 -0700
Subject: [PATCH 178/183] refactor(service): users/auth: change property on
 return types

---
 service/src/ingress/ingress.app.api.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service/src/ingress/ingress.app.api.ts b/service/src/ingress/ingress.app.api.ts
index b96544331..b4f3c71f2 100644
--- a/service/src/ingress/ingress.app.api.ts
+++ b/service/src/ingress/ingress.app.api.ts
@@ -42,7 +42,7 @@ export interface AdmitFromIdentityProviderRequest {
 
 export interface AdmitFromIdentityProviderResult {
   mageAccount: UserExpanded
-  admissionToken: string
+  idpAuthenticationToken: string
 }
 
 /**

From 484b3ecb952255a36c5419614aca52153e5f9ffb Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sat, 30 Nov 2024 23:27:40 -0700
Subject: [PATCH 179/183] refactor(service): users/auth: user operations types

---
 .../users/adapters.users.controllers.web.ts   | 132 ++++++++++++++++--
 service/src/app.api/users/app.api.users.ts    |  13 +-
 .../ingress.adapters.controllers.web.ts       |  87 ++++++++++--
 3 files changed, 206 insertions(+), 26 deletions(-)

diff --git a/service/src/adapters/users/adapters.users.controllers.web.ts b/service/src/adapters/users/adapters.users.controllers.web.ts
index 11c053936..46715da7f 100644
--- a/service/src/adapters/users/adapters.users.controllers.web.ts
+++ b/service/src/adapters/users/adapters.users.controllers.web.ts
@@ -1,16 +1,62 @@
 import express from 'express'
-import { SearchUsers, UserSearchRequest } from '../../app.api/users/app.api.users'
+import { SearchUsers, UserSearchRequest, CreateUserOperation } from '../../app.api/users/app.api.users'
 import { WebAppRequestFactory } from '../adapters.controllers.web'
-import { calculateLinks } from '../../entities/entities.global'
+import { calculatePagingLinks } from '../../entities/entities.global'
+import { defaultHandler as upload } from '../../upload'
+import { Phone, UserExpanded, UserIcon } from '../../entities/users/entities.users'
+import { invalidInput, InvalidInputError } from '../../app.api/app.api.errors'
+import { AppRequest } from '../../app.api/app.api.global'
 
 export interface UsersAppLayer {
+  createUser: CreateUserOperation
   searchUsers: SearchUsers
 }
 
-export function UsersRoutes(app: UsersAppLayer, createAppRequest: WebAppRequestFactory): express.Router {
+export function UsersRoutes(app: UsersAppLayer, createAppRequest: WebAppRequestFactory<AppRequest<UserExpanded>>): express.Router {
 
   const routes = express.Router()
 
+  routes.route('/')
+    .post(
+      access.authorize('CREATE_USER'),
+      upload.fields([ { name: 'avatar' }, { name: 'icon' } ]),
+      function (req, res, next) {
+        const accountForm = validateAccountForm(req)
+        if (accountForm instanceof Error) {
+          return next(accountForm)
+        }
+        const iconAttrs = parseIconUpload(req)
+        if (iconAttrs instanceof Error) {
+          return next(iconAttrs)
+        }
+        const user = {
+          username: accountForm.username,
+          roleId: accountForm.roleId,
+          active: true, // Authorized to update users, activate account by default
+          displayName: accountForm.displayName,
+          email: accountForm.email,
+          phones: accountForm.phones,
+          authentication: {
+            type: 'local',
+            password: accountForm.password,
+            authenticationConfiguration: {
+              name: 'local'
+            }
+          }
+        }
+        const files = req.files as Record<string, Express.Multer.File[]> || {}
+        const [ avatar ] = files.avatar || []
+        const [ icon ] = files.icon || []
+
+        // TODO: users-next
+        app.createUser()
+        new api.User().create(user, { avatar, icon }).then(newUser => {
+          newUser = userTransformer.transform(newUser, { path: req.getRoot() });
+          res.json(newUser);
+        }).catch(err => next(err));
+      }
+    );
+
   routes.route('/search')
     .get(async (req, res, next) => {
       const userSearch: UserSearchRequest['userSearch'] = {
@@ -26,24 +72,88 @@ export function UsersRoutes(app: UsersAppLayer, createAppRequest: WebAppRequestF
           'enabled' in req.query
             ? /^true$/i.test(String(req.query.enabled))
             : undefined
-      };
-
+      }
       const appReq = createAppRequest(req, { userSearch })
       const appRes = await app.searchUsers(appReq)
        if (appRes.success) {
-         const links = calculateLinks(
+         const links = calculatePagingLinks(
            { pageSize: userSearch.pageSize, pageIndex: userSearch.pageIndex },
            appRes.success.totalCount
-         );
-
+         )
          const responseWithLinks = {
            ...appRes.success,
            links
-         };
-
-         return res.json(responseWithLinks);
+         }
+         return res.json(responseWithLinks)
        }
       next(appRes.error)
     })
+
   return routes
+}
+
+interface AccountForm {
+  username: string
+  password: string
+  displayName: string
+  email?: string
+  phones?: Phone[]
+  roleId: string
+}
+
+function validateAccountForm(req: express.Request): AccountForm | InvalidInputError {
+  const username = req.body.username
+  if (typeof username !== 'string' || username.length === 0) {
+    return invalidInput('username is required')
+  }
+  const displayName = req.body.displayName
+  if (typeof displayName !== 'string' || displayName.length === 0) {
+    return invalidInput('displayName is required')
+  }
+  const email = req.body.email
+  if (typeof email === 'string') {
+    const emailRegex = /^[^\s@]+@[^\s@]+\./
+    if (!emailRegex.test(email)) {
+      return invalidInput('invalid email')
+    }
+  }
+  const formPhone = req.body.phone
+  const phones: Phone[] = typeof formPhone === 'string' ?
+    [ { type: 'Main', number: formPhone } ] : []
+  const password = req.body.password
+  if (typeof password !== 'string') {
+    return invalidInput('password is required')
+  }
+  const roleId = req.body.roleId
+  if (typeof roleId !== 'string') {
+    return invalidInput('roleId is required')
+  }
+  return {
+    username: username.trim(),
+    password,
+    displayName,
+    email,
+    phones,
+    roleId,
+  }
+}
+
+function parseIconUpload(req: express.Request): UserIcon | InvalidInputError {
+  const formIconAttrs = req.body.iconMetadata || {} as any
+  const iconAttrs: Partial<UserIcon> =
+    typeof formIconAttrs === 'string' ?
+      JSON.parse(formIconAttrs) :
+      formIconAttrs
+  const files = req.files as Record<string, Express.Multer.File[]> || { icon: [] }
+  const [ iconFile ] = files.icon || []
+  if (iconFile) {
+    if (!iconAttrs.type) {
+      iconAttrs.type = UserIconType.Upload
+    }
+    if (iconAttrs.type !== 'create' && iconAttrs.type !== 'upload') {
+      // TODO: does this really matter? just take the uploaded image
+      return invalidInput(`invalid icon type: ${iconAttrs.type}`)
+    }
+  }
+  return { type: UserIconType.None }
 }
\ No newline at end of file
diff --git a/service/src/app.api/users/app.api.users.ts b/service/src/app.api/users/app.api.users.ts
index 4b6fe9fd0..925f23510 100644
--- a/service/src/app.api/users/app.api.users.ts
+++ b/service/src/app.api/users/app.api.users.ts
@@ -1,9 +1,17 @@
 import { AppResponse, AppRequest, AppRequestContext } from '../app.api.global'
-import { PermissionDeniedError, InvalidInputError } from '../app.api.errors'
+import { EntityNotFoundError, InvalidInputError, PermissionDeniedError } from '../app.api.errors'
 import { PageOf, PagingParameters } from '../../entities/entities.global'
-import { User } from '../../entities/users/entities.users'
+import { User, UserExpanded, UserId } from '../../entities/users/entities.users'
 
 
+export interface CreateUserRequest extends AppRequest {
+
+}
+
+export interface CreateUserOperation {
+  (req: CreateUserRequest): Promise<AppResponse<UserExpanded, PermissionDeniedError | InvalidInputError>>
+}
+
 export interface UserSearchRequest extends AppRequest {
   userSearch: PagingParameters & {
     nameOrContactTerm?: string | undefined,
@@ -22,6 +30,7 @@ export type UserSearchResult = Pick<User, 'id' | 'username' | 'displayName' | 'e
 export interface SearchUsers {
   (req: UserSearchRequest): Promise<AppResponse<PageOf<UserSearchResult>, PermissionDeniedError>>
 }
+
 export interface ReadMyAccountRequest extends AppRequest<User> {}
 
 export interface ReadMyAccountOperation {
diff --git a/service/src/ingress/ingress.adapters.controllers.web.ts b/service/src/ingress/ingress.adapters.controllers.web.ts
index 40650a037..1eaf13723 100644
--- a/service/src/ingress/ingress.adapters.controllers.web.ts
+++ b/service/src/ingress/ingress.adapters.controllers.web.ts
@@ -1,13 +1,14 @@
 import express from 'express'
 import svgCaptcha from 'svg-captcha'
-import { Authenticator } from 'passport'
-import { Strategy as BearerStrategy } from 'passport-http-bearer'
+import passport from 'passport'
+import bearer from 'passport-http-bearer'
 import { defaultHashUtil } from '../utilities/password-hashing'
 import { JWTService, Payload, TokenVerificationError, VerificationErrorReason, TokenAssertion } from './verification'
 import { invalidInput, InvalidInputError, MageError } from '../app.api/app.api.errors'
 import { IdentityProvider } from './ingress.entities'
 import { AdmitFromIdentityProviderOperation, EnrollMyselfOperation, EnrollMyselfRequest } from './ingress.app.api'
 import { IngressProtocolWebBinding, IngressResponseType } from './ingress.protocol.bindings'
+import { User, UserRepository } from '../entities/users/entities.users'
 
 declare module 'express-serve-static-core' {
   interface Request {
@@ -50,7 +51,30 @@ export type IngressRoutes = {
   idpAdmission: express.Router
 }
 
-export function CreateIngressRoutes(ingressApp: IngressUseCases, idpCache: IngressProtocolWebBindingCache, tokenService: JWTService, passport: Authenticator): IngressRoutes {
+/**
+ * Register a `BearerStrategy` that expects a JWT in the `Authorization` header that contains the
+ * {@link TokenAssertion.Authenticated} claim.  The claim indicates the subject has authenticated with an IDP and can
+ * continue the ingress process.  Decode and verify the JWT signature, retrieve the `User` for the JWT subject, and set
+ * `Request.user`.
+ */
+function createIdpAuthenticationTokenVerificationStrategy(passport: passport.Authenticator, verificationService: JWTService, userRepo: UserRepository): bearer.Strategy {
+  return new bearer.Strategy(async function(token, done: (error: any, user?: User) => any) {
+    try {
+      const expectation: Payload = { assertion: TokenAssertion.Authenticated, subject: null, expiration: null }
+      const payload = await verificationService.verifyToken(token, expectation)
+      const user = payload.subject ? await userRepo.findById(payload.subject) : null
+      if (user) {
+        return done(null, user)
+      }
+      done(new Error(`user id ${payload.subject} not found for transient token ${String(payload)}`))
+    }
+    catch (err) {
+      done(err)
+    }
+  })
+}
+
+export function CreateIngressRoutes(ingressApp: IngressUseCases, idpCache: IngressProtocolWebBindingCache, tokenService: JWTService, passport: passport.Authenticator): IngressRoutes {
 
   const routeToIdp = express.Router()
     .all('/',
@@ -81,28 +105,29 @@ export function CreateIngressRoutes(ingressApp: IngressUseCases, idpCache: Ingre
         if (admission.error) {
           return next(admission.error)
         }
-        const { admissionToken, mageAccount } = admission.success
+        const { idpAuthenticationToken, mageAccount } = admission.success
         if (idpBinding.ingressResponseType === IngressResponseType.Direct) {
-          return res.json({ user: mageAccount, token: admissionToken })
+          return res.json({ user: mageAccount, token: idpAuthenticationToken })
         }
         if (idpAdmission.flowState === UserAgentType.MobileApp) {
           if (mageAccount.active && mageAccount.enabled) {
-            return res.redirect(`mage://app/authentication?token=${admissionToken}`)
+            return res.redirect(`mage://app/authentication?token=${idpAuthenticationToken}`)
           }
           else {
             return res.redirect(`mage://app/invalid_account?active=${mageAccount.active}&enabled=${mageAccount.enabled}`)
           }
         }
         else if (idpAdmission.flowState === UserAgentType.WebApp) {
-          return res.render('authentication', { host: req.getRoot(), success: true, login: { token: admissionToken, user: mageAccount } })
+          return res.render('authentication', { host: req.getRoot(), success: true, login: { token: idpAuthenticationToken, user: mageAccount } })
         }
         return res.status(500).send('invalid authentication state')
       }) as express.ErrorRequestHandler
     )
 
   // TODO: mount to /auth
-  const idpAdmission = express.Router()
-  idpAdmission.use('/:identityProviderName',
+  const admission = express.Router()
+
+  admission.use('/:identityProviderName',
     async (req, res, next) => {
       const idpName = req.params.identityProviderName
       const idpBindingEntry = await idpCache.idpWebBindingForIdpName(idpName)
@@ -117,7 +142,43 @@ export function CreateIngressRoutes(ingressApp: IngressUseCases, idpCache: Ingre
     routeToIdp
   )
 
-  // TODO: mount to /api/users/signups
+  const verifyIdpAuthenticationTokenStrategy = createIdpAuthenticationTokenVerificationStrategy(passport, tokenService, userRepo)
+  admission.post('/token',
+    passport.authenticate(verifyIdpAuthenticationTokenStrategy),
+    async (req, res, next) => {
+      deviceProvisioning.check()
+      const options = {
+        userAgent: req.headers['user-agent'],
+        appVersion: req.body.appVersion
+      }
+      /*
+      TODO: users-next
+      insert a new login record for the user and start a new session
+      retrieve the server api descriptor
+      add the available identity providers to the api descriptor
+      return a json object shaped as below
+      */
+      // new api.User().login(req.user, req.provisionedDevice, options, function (err, session) {
+      //   if (err) return next(err);
+
+      //   authenticationApiAppender.append(config.api).then(api => {
+      //     res.json({
+      //       token: session.token,
+      //       expirationDate: session.expirationDate,
+      //       user: userTransformer.transform(req.user, { path: req.getRoot() }),
+      //       device: req.provisionedDevice,
+      //       api: api
+      //     });
+      //   }).catch(err => {
+      //     next(err);
+      //   });
+      // });
+
+      // req.session = null;
+    }
+  )
+
+  // TODO: users-next: mount to /api/users/signups
   const localEnrollment = express.Router()
   localEnrollment.route('/signups')
     .post(async (req, res, next) => {
@@ -146,7 +207,7 @@ export function CreateIngressRoutes(ingressApp: IngressUseCases, idpCache: Ingre
       }
     })
 
-  const captchaBearer = new BearerStrategy((token, done) => {
+  const captchaBearer = new bearer.Strategy((token, done) => {
     const expectation = {
       subject: null,
       expiration: null,
@@ -157,7 +218,7 @@ export function CreateIngressRoutes(ingressApp: IngressUseCases, idpCache: Ingre
       .catch(err => done(err))
   })
 
-  // TODO: mount to /api/users/signups/verifications
+  // TODO: users-next: mount to /api/users/signups/verifications
   localEnrollment.route('/signups/verifications')
     .post(
       async (req, res, next) => {
@@ -208,7 +269,7 @@ export function CreateIngressRoutes(ingressApp: IngressUseCases, idpCache: Ingre
       }
     )
 
-  return { localEnrollment, idpAdmission }
+  return { localEnrollment, idpAdmission: admission }
 }
 
 function validateEnrollment(input: any): Omit<EnrollMyselfRequest, 'username'> | InvalidInputError {

From 3270eba0be9266e9aef92ec9874c2e00ebf60386 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sun, 1 Dec 2024 00:06:56 -0700
Subject: [PATCH 180/183] refactor(service): users/auth: add todo note

---
 service/src/ingress/index.ts | 39 +-----------------------------------
 1 file changed, 1 insertion(+), 38 deletions(-)

diff --git a/service/src/ingress/index.ts b/service/src/ingress/index.ts
index b355346ac..33c3bf9ba 100644
--- a/service/src/ingress/index.ts
+++ b/service/src/ingress/index.ts
@@ -15,46 +15,9 @@ const authenticationApiAppender = require('../utilities/authenticationApiAppende
 const AuthenticationConfiguration = require('../models/authenticationconfiguration')
 const SecurePropertyAppender = require('../security/utilities/secure-property-appender');
 
-
 /**
- * Register the route to generate an API access token, the final step in the ingress process after enrollment,
- * authentication.  This step includes provisioning a device based on the configured policy.
+ * TODO: users-next: this module should go away. this remains for now as a reference to migrate legacy logic to new architecture
  */
-function registerDeviceVerificationAndTokenGenerationEndpoint(routes: express.Router, passport: passport.Authenticator, deviceProvisioning: ProvisionStatic, sessionRepo: SessionRepository) {
-  routes.post('/auth/token',
-    passport.authenticate(VerifyIdpAuthenticationToken),
-    async (req, res, next) => {
-      deviceProvisioning.check()
-      const options = {
-        userAgent: req.headers['user-agent'],
-        appVersion: req.body.appVersion
-      }
-      // TODO: users-next
-      new api.User().login(req.user, req.provisionedDevice, options, function (err, session) {
-        if (err) return next(err);
-
-        authenticationApiAppender.append(config.api).then(api => {
-          res.json({
-            token: session.token,
-            expirationDate: session.expirationDate,
-            user: userTransformer.transform(req.user, { path: req.getRoot() }),
-            device: req.provisionedDevice,
-            api: api
-          });
-        }).catch(err => {
-          next(err);
-        });
-      });
-
-      req.session = null;
-    }
-  );
-}
-
-function registerLocalAuthenticationProtocol(): void {
-
-}
-
 
 export class AuthenticationInitializer {
   static tokenService = new JWTService(crypto.randomBytes(64).toString('hex'), 'urn:mage');

From 67409979004cd373814a5cc598869b4eb1d88d3a Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sun, 1 Dec 2024 07:22:21 -0700
Subject: [PATCH 181/183] refactor(service): users/auth: remove unnecessary
 express session middleware; mage does not require a session cookie

---
 service/src/express.js              |  4 ----
 service/src/ingress/ingress.main.ts | 14 --------------
 2 files changed, 18 deletions(-)

diff --git a/service/src/express.js b/service/src/express.js
index 5a49fe017..b5cd6a254 100644
--- a/service/src/express.js
+++ b/service/src/express.js
@@ -29,9 +29,6 @@ app.use(function(req, res, next) {
   return next();
 });
 
-const secret = crypto.randomBytes(64).toString('hex');
-app.use(session({ secret }));
-
 app.set('config', config);
 app.enable('trust proxy');
 
@@ -44,7 +41,6 @@ app.use(
     express.urlencoded( { ...jsonOptions, extended: true }));
 
 app.use(passport.initialize());
-app.use(passport.session());
 app.get('/api/docs/openapi.yaml', async function(req, res) {
   const docPath = path.resolve(__dirname, 'docs', 'openapi.yaml');
   fs.readFile(docPath, (err, contents) => {
diff --git a/service/src/ingress/ingress.main.ts b/service/src/ingress/ingress.main.ts
index e23090e95..15fff8948 100644
--- a/service/src/ingress/ingress.main.ts
+++ b/service/src/ingress/ingress.main.ts
@@ -68,20 +68,6 @@ export async function initializeIngress(
   provisioning: provision.ProvisionStatic,
   passport: passport.Authenticator,
 ): Promise<express.Router> {
-  // TODO: users-next: these serialization functions are probably no longer necessary
-  passport.serializeUser((user, done) => done(null, user.id))
-  passport.deserializeUser(async (id, done) => {
-    try {
-      const user = await userRepo.findById(String(id))
-      done(null, user)
-    }
-    catch (err) {
-      done(err)
-    }
-  })
-  const routes = express.Router()
-  registerAuthenticatedBearerTokenHandling(passport, sessionRepo, userRepo)
-  return routes
 }
 
 /**

From 9eb6d42a41ca3353f7b7cf1ee7ba87cfa182e29b Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sun, 1 Dec 2024 07:24:00 -0700
Subject: [PATCH 182/183] refactor(service): users/auth: remove unnecessary
 passport initialize call

---
 service/src/express.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/service/src/express.js b/service/src/express.js
index b5cd6a254..2665b8d47 100644
--- a/service/src/express.js
+++ b/service/src/express.js
@@ -40,7 +40,6 @@ app.use(
     express.json(jsonOptions),
     express.urlencoded( { ...jsonOptions, extended: true }));
 
-app.use(passport.initialize());
 app.get('/api/docs/openapi.yaml', async function(req, res) {
   const docPath = path.resolve(__dirname, 'docs', 'openapi.yaml');
   fs.readFile(docPath, (err, contents) => {

From 7c7bbb3d590e2d7c15a4cd4e3b4b7962bb0a4b57 Mon Sep 17 00:00:00 2001
From: "Robert St. John" <restjohn@users.noreply.github.com>
Date: Sun, 1 Dec 2024 07:25:33 -0700
Subject: [PATCH 183/183] refactor(service): users/auth: add todo notes

---
 service/src/express.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/service/src/express.js b/service/src/express.js
index 2665b8d47..d7d91fd0a 100644
--- a/service/src/express.js
+++ b/service/src/express.js
@@ -54,10 +54,11 @@ app.use('/private',
   express.static(path.join(__dirname, 'private')));
 
 // Configure authentication
+// TODO: users-next: remove this and initialize in main app module
 const authentication = AuthenticationInitializer.initialize(app, passport, provision);
 
 // Configure routes
-// TODO: don't pass authentication to other routes, but enforce authentication ahead of adding route modules
+// TODO: users-next: don't pass authentication to other routes, but enforce authentication ahead of adding route modules
 require('./routes')(app, { authentication });
 
 // Express requires a 4 parameter function callback, do not remove unused next parameter