lots of misc clean up

Author: ChrisRackauckas

.github/workflows/CI.yml

@@ -14,9 +14,8 @@ jobs:
       matrix:
         group:
           - NNODE
-          - NNPDEHan
-          - NNPDENS
-          - NNPDE
+          - NNPDE1
+          - NNPDE2
           - AdaptiveLoss
           - NeuralAdapter
           - IntegroDiff

CITATION.bib

@@ -1,13 +1,10 @@
-@article{DifferentialEquations.jl-2017,
- author = {Rackauckas, Christopher and Nie, Qing},
- doi = {10.5334/jors.151},
- journal = {The Journal of Open Research Software},
- keywords = {Applied Mathematics},
- note = {Exported from https://app.dimensions.ai on 2019/05/05},
- number = {1},
- pages = {},
- title = {DifferentialEquations.jl – A Performant and Feature-Rich Ecosystem for Solving Differential Equations in Julia},
- url = {https://app.dimensions.ai/details/publication/pub.1085583166 and http://openresearchsoftware.metajnl.com/articles/10.5334/jors.151/galley/245/download/},
- volume = {5},
- year = {2017}
-}
+@misc{https://doi.org/10.48550/arxiv.2107.09443,
+  doi = {10.48550/ARXIV.2107.09443},
+  url = {https://arxiv.org/abs/2107.09443},
+  author = {Zubov, Kirill and McCarthy, Zoe and Ma, Yingbo and Calisto, Francesco and Pagliarino, Valerio and Azeglio, Simone and Bottero, Luca and Luján, Emmanuel and Sulzer, Valentin and Bharambe, Ashutosh and Vinchhi, Nand and Balakrishnan, Kaushik and Upadhyay, Devesh and Rackauckas, Chris},
+  keywords = {Mathematical Software (cs.MS), Symbolic Computation (cs.SC), FOS: Computer and information sciences, FOS: Computer and information sciences},  
+  title = {NeuralPDE: Automating Physics-Informed Neural Networks (PINNs) with Error Approximations},
+  publisher = {arXiv},
+  year = {2021},
+  copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International}
+}

docs/make.jl

@@ -8,7 +8,14 @@ makedocs(
     clean=true,
     doctest=false,
     modules=[NeuralPDE],
-
+    strict=[
+        :doctest, 
+        :linkcheck, 
+        :parse_error,
+        :example_block,
+        # Other available options are
+        # :autodocs_block, :cross_references, :docs_block, :eval_block, :example_block, :footnote, :meta_block, :missing_docs, :setup_block
+    ],
     format=Documenter.HTML(  analytics = "UA-90474609-3",
                              assets=["assets/favicon.ico"],
                              canonical="https://neuralpde.sciml.ai/stable/"),

docs/src/examples/ode.md

@@ -1,19 +1,74 @@
-# Solving ODEs with Neural Networks
+# Solving ODEs with Physics-Informed Neural Networks
 
-The following is an example of solving a DifferentialEquations.jl
-`ODEProblem` with a neural network using the physics-informed neural
-networks approach specialized to 1-dimensional PDEs (ODEs).
+!!! note
 
-```julia
-using Flux, OptimizationOptimisers, OptimizationOptimJL
+    It is highly recommended you first read the [solving ordinary differential
+    equations with DifferentialEquations.jl tutorial](https://diffeq.sciml.ai/stable/tutorials/ode_example/)
+    before reading this tutorials.
+
+This tutorial is an introduction to using physics-informed neural networks (PINNs)
+for solving ordinary differential equations (ODEs). In contrast to the later
+parts of this documentation which use the symbolic interface, here we will focus on 
+the simplified `NNODE` which uses the `ODEProblem` specification for the ODE.
+Mathematically the `ODEProblem` defines a problem:
+
+```math
+u' = f(u,p,t)
+```
+
+for ``t \in (t_0,t_f)`` with an initial condition ``u(t_0) = u_0``. With physics-informed
+neural networks, we choose a neural network architecture `NN` to represent the solution `u`
+and seek to find parameters `p` such that `NN' = f(NN,p,t)` for all points in the domain.
+When this is satisfied sufficiently closely, then `NN` is thus a solution to the differential
+equation.
+
+## Solving an ODE with NNODE
+
+Let's solve the simple ODE:
+
+```math
+u' = \cos(2\pi t)
+```
+
+for ``t \in (0,1)`` and ``u_0 = 0`` with `NNODE`. First we define the `ODEProblem` as we would
+with any other DifferentialEquations.jl solver. This looks like:
+
+```@example nnode1
 using NeuralPDE
-# Run a solve on scalars
-linear = (u, p, t) -> cos(2pi * t)
+
+f(u, p, t) = cos(2pi * t)
 tspan = (0.0f0, 1.0f0)
 u0 = 0.0f0
 prob = ODEProblem(linear, u0, tspan)
+```
+
+Now, to define the `NNODE` solver, we must choose a neural network architecture. To do this, we
+will use the [Flux.jl library](https://fluxml.ai/) to define a multilayer perceptron (MLP)
+with one hidden layer of 5 nodes and a sigmoid activation function. This looks like:
+
+```@example nnode1
 chain = Flux.Chain(Dense(1, 5, σ), Dense(5, 1))
-opt = Flux.ADAM(0.1, (0.9, 0.95))
-@time sol = solve(prob, NeuralPDE.NNODE(chain, opt), dt=1 / 20f0, verbose=true,
+```
+
+Now we must choose an optimizer to define the `NNODE` solver. A common choice is `ADAM`, with
+a tunable rate parameter which we will set to `0.1`. In general, this rate parameter should be
+decreased if the solver's loss tends to be unsteady (sometimes rise "too much"), but should be
+as large as possible for efficnecy. Thus the definition of the `NNODE` solver is as follows:
+
+```@example nnode1
+opt = Flux.ADAM(0.1)
+alg = NeuralPDE.NNODE(chain, opt)
+```
+
+Once these pieces are together, we call `solve` just like with any other `ODEProblem` solver.
+Let's turn on `verbose` so we can see the loss over time during the training process:
+
+```@example nnode1
+sol = solve(prob, NeuralPDE.NNODE(chain, opt), dt=1 / 20f0, verbose=true,
             abstol=1e-10, maxiters=200)
 ```
+
+And that's it: the neural network solution was computed by training the neural network and
+returned in the standard DifferentialEquations.jl `ODESolution` format. For more information
+on handling the solution, consult 
+[the DifferentialEquations.jl solution handling section](https://diffeq.sciml.ai/stable/basics/solution/)

docs/src/index.md

@@ -2,24 +2,32 @@
 
 [NeuralPDE.jl](https://github.com/SciML/NeuralPDE.jl)
 NeuralPDE.jl is a solver package which consists of neural network solvers for
-partial differential equations using physics-informed neural networks (PINNs). This package utilizes
-neural stochastic differential equations to solve PDEs at a greatly increased generality 
-compared with classical methods.
+partial differential equations using physics-informed neural networks (PINNs).
 
 ## Features
 
-- Physics-Informed Neural Networks for automated PDE solving
-- Deep-learning-based solvers for optimal stopping time and Kolmogorov backwards equations
+- Physics-Informed Neural Networks for ODE, SDE, RODE, and PDE solving
+- Ability to define extra loss functions to mix xDE solving with data fitting (scientific machine learning)
+- Automated construction of Physics-Informed loss functions from a high level symbolic interface
+- Sophisticated techniques like quadrature training strategies, adaptive loss functions, and neural adapters
+  to accelerate training
+- Integrated logging suite for handling connections to TensorBoard
+- Handling of (partial) integro-differential equations and various stochastic equations
+- Specialized forms for solving `ODEProblem`s with neural networks
 
 ## Citation
 
 If you use NeuralPDE.jl in your research, please cite [this paper](https://arxiv.org/abs/2107.09443):
 
 ```tex
-@article{zubov2021neuralpde,
-  title={NeuralPDE: Automating Physics-Informed Neural Networks (PINNs) with Error Approximations},
-  author={Zubov, Kirill and McCarthy, Zoe and Ma, Yingbo and Calisto, Francesco and Pagliarino, Valerio and Azeglio, Simone and Bottero, Luca and Luj{\'a}n, Emmanuel and Sulzer, Valentin and Bharambe, Ashutosh and others},
-  journal={arXiv preprint arXiv:2107.09443},
-  year={2021}
+@misc{https://doi.org/10.48550/arxiv.2107.09443,
+  doi = {10.48550/ARXIV.2107.09443},
+  url = {https://arxiv.org/abs/2107.09443},
+  author = {Zubov, Kirill and McCarthy, Zoe and Ma, Yingbo and Calisto, Francesco and Pagliarino, Valerio and Azeglio, Simone and Bottero, Luca and Luján, Emmanuel and Sulzer, Valentin and Bharambe, Ashutosh and Vinchhi, Nand and Balakrishnan, Kaushik and Upadhyay, Devesh and Rackauckas, Chris},
+  keywords = {Mathematical Software (cs.MS), Symbolic Computation (cs.SC), FOS: Computer and information sciences, FOS: Computer and information sciences},  
+  title = {NeuralPDE: Automating Physics-Informed Neural Networks (PINNs) with Error Approximations},
+  publisher = {arXiv},
+  year = {2021},
+  copyright = {Creative Commons Attribution Non Commercial Share Alike 4.0 International}
 }
 ```

docs/src/solvers/ode.md

@@ -1,20 +1,5 @@
 # ODE-Specialized Physics-Informed Neural Solver
 
-The ODE-specialized physics-informed neural network (PINN) solver is a
-method for the [DifferentialEquations.jl common interface](https://diffeq.sciml.ai/latest/)
-of `ODEProblem`, which generates the solution via a neural network.
-Thus the standard [ODEProblem](https://diffeq.sciml.ai/latest/types/ode_types/)
-is used, but a new algorithm, `NNODE`, is used to solve the problem.
-
-The algorithm type is:
-
-```julia
-nnode(chain,opt)
-```
-
-where `chain` is a DiffEqFlux `sciml_train`-compatible Chain or FastChain
-representing a neural network, and `opt` is an optimization method
-for `sciml_train`. For more details, see [the DiffEqFlux documentation
-on `sciml_train`](https://diffeqflux.sciml.ai/dev/).
-
-[Lagaris, Isaac E., Aristidis Likas, and Dimitrios I. Fotiadis. "Artificial neural networks for solving ordinary and partial differential equations." IEEE Transactions on Neural Networks 9, no. 5 (1998): 987-1000.](https://arxiv.org/pdf/physics/9705023.pdf)
+```@docs
+NNODE
+```

src/NeuralPDE.jl

@@ -141,36 +141,6 @@ function Base.show(io::IO, A::ParamKolmogorovPDEProblem)
   show(io , A.g)
 end
 
-"""
-Algorithm for solving differential equation using a neural network.
-Arguments:
-* `chain`: A Chain neural network
-* `opt`: The optimizer to train the neural network. Defaults to `BFGS()`.
-* `initθ`: The initial parameter of the neural network.
-* `autodiff`: The switch between automatic and numerical differentiation for
-              the PDE operators. The reverse mode of the loss function is always AD.
-"""
-struct NNODE{C,O,P,K} <: NeuralPDEAlgorithm
-    chain::C
-    opt::O
-    initθ::P
-    autodiff::Bool
-    kwargs::K
-end
-function NNODE(chain,opt=Optim.BFGS(),init_params = nothing;autodiff=false,kwargs...)
-    if init_params === nothing
-        if chain isa FastChain
-            initθ = DiffEqFlux.initial_params(chain)
-        else
-            initθ,re  = Flux.destructure(chain)
-        end
-    else
-        initθ = init_params
-    end
-    NNODE(chain,opt,initθ,autodiff,kwargs)
-end
-
-
 include("training_strategies.jl")
 include("adaptive_losses.jl")
 include("ode_solve.jl")

src/ode_solve.jl

@@ -1,3 +1,63 @@
+"""
+```julia
+NNODE(chain,opt=Optim.BFGS(),init_params = nothing;
+                          autodiff=false,kwargs...)
+```
+
+Algorithm for solving ordinary differential equations using a neural network. This is a specialization
+of the physics-informed neural network which is used as a solver for a standard `ODEProblem`.
+
+## Positional Arguments
+
+* `chain`: A Chain neural network
+* `opt`: The optimizer to train the neural network. Defaults to `BFGS()`.
+* `initθ`: The initial parameter of the neural network.
+
+## Keyword Arguments
+
+* `autodiff`: The switch between automatic and numerical differentiation for
+              the PDE operators. The reverse mode of the loss function is always
+              automatic differentation (via Zygote), this is only for the derivative
+              in the loss function (the derivative with respect to time).
+
+## Example
+
+```julia
+f(u,p,t) = cos(2pi*t)
+tspan = (0.0f0, 1.0f0)
+u0 = 0.0f0
+prob = ODEProblem(linear, u0 ,tspan)
+chain = Flux.Chain(Dense(1,5,σ),Dense(5,1))
+opt = Flux.ADAM(0.1, (0.9, 0.95))
+sol = solve(prob, NeuralPDE.NNODE(chain,opt), dt=1/20f0, verbose = true,
+            abstol=1e-10, maxiters = 200)
+```
+
+## References
+
+Lagaris, Isaac E., Aristidis Likas, and Dimitrios I. Fotiadis. "Artificial neural networks for solving 
+ordinary and partial differential equations." IEEE Transactions on Neural Networks 9, no. 5 (1998): 987-1000.
+"""
+struct NNODE{C,O,P,K} <: NeuralPDEAlgorithm
+    chain::C
+    opt::O
+    initθ::P
+    autodiff::Bool
+    kwargs::K
+end
+function NNODE(chain,opt=Optim.BFGS(),init_params = nothing;autodiff=false,kwargs...)
+    if init_params === nothing
+        if chain isa FastChain
+            initθ = DiffEqFlux.initial_params(chain)
+        else
+            initθ,re  = Flux.destructure(chain)
+        end
+    else
+        initθ = init_params
+    end
+    NNODE(chain,opt,initθ,autodiff,kwargs)
+end
+
 function DiffEqBase.solve(
     prob::DiffEqBase.AbstractODEProblem,
     alg::NNODE,
@@ -28,19 +88,20 @@ function DiffEqBase.solve(
     if chain isa FastChain
         #The phi trial solution
         if u0 isa Number
-            phi = (t,θ) -> u0 + (t-tspan[1])*first(chain(adapt(DiffEqBase.parameterless_type(θ),[t]),θ))
+            phi = (t,θ) -> u0 + (t-tspan[1]) * first(chain(adapt(parameterless_type(θ),[t]),θ))
         else
-            phi = (t,θ) -> u0 + (t-tspan[1]) * chain(adapt(DiffEqBase.parameterless_type(θ),[t]),θ)
+            phi = (t,θ) -> u0 + (t-tspan[1]) * chain(adapt(parameterless_type(θ),[t]),θ)
         end
     else
         _,re  = Flux.destructure(chain)
         #The phi trial solution
         if u0 isa Number
-            phi = (t,θ) -> u0 + (t-tspan[1])*first(re(θ)(adapt(DiffEqBase.parameterless_type(θ),[t])))
+            phi = (t,θ) -> u0 + (t-tspan[1])*first(re(θ)(adapt(parameterless_type(θ),[t])))
         else
-            phi = (t,θ) -> u0 + (t-tspan[1]) * re(θ)(adapt(DiffEqBase.parameterless_type(θ),[t]))
+            phi = (t,θ) -> u0 + (t-tspan[1]) * re(θ)(adapt(parameterless_type(θ),[t]))
         end
     end
+
     try
         phi(t0 , initθ)
     catch err