Dual NNUE with L1-128 smallnet

Credit goes to @mstembera for: - writing the code enabling dual NNUE: #4898 - the idea of trying L1-128 trained exclusively on high simple eval positions The L1-128 smallnet is: - epoch 399 of a single-stage training from scratch - trained only on positions from filtered data with high material difference - defined by abs(simple_eval) > 1000 ```yaml experiment-name: 128--S1-only-hse-v2 training-dataset: - /data/hse/S3/dfrc99-16tb7p-eval-filt-v2.min.high-simple-eval-1k.binpack - /data/hse/S3/leela96-filt-v2.min.high-simple-eval-1k.binpack - /data/hse/S3/test80-apr2022-16tb7p.min.high-simple-eval-1k.binpack - /data/hse/S7/test60-2020-2tb7p.v6-3072.high-simple-eval-1k.binpack - /data/hse/S7/test60-novdec2021-12tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test77-nov2021-2tb7p.v6-3072.min.high-simple-eval-1k.binpack - /data/hse/S7/test77-dec2021-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test77-jan2022-2tb7p.high-simple-eval-1k.binpack - /data/hse/S7/test78-jantomay2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test78-juntosep2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test79-apr2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test79-may2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack # T80 2022 - /data/hse/S7/test80-may2022-16tb7p.high-simple-eval-1k.binpack - /data/hse/S7/test80-jun2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test80-jul2022-16tb7p.v6-dd.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-aug2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test80-sep2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test80-oct2022-16tb7p.v6-dd.high-simple-eval-1k.binpack - /data/hse/S7/test80-nov2022-16tb7p-v6-dd.min.high-simple-eval-1k.binpack # T80 2023 - /data/hse/S7/test80-jan2023-3of3-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test80-feb2023-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test80-mar2023-2tb7p.v6-sk16.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-apr2023-2tb7p-filter-v6-sk16.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-may2023-2tb7p.v6.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-jun2023-2tb7p.v6-3072.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-jul2023-2tb7p.v6-3072.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-aug2023-2tb7p.v6.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-sep2023-2tb7p.high-simple-eval-1k.binpack - /data/hse/S7/test80-oct2023-2tb7p.high-simple-eval-1k.binpack start-from-engine-test-net: False nnue-pytorch-branch: linrock/nnue-pytorch/L1-128 engine-test-branch: linrock/Stockfish/L1-128-nolazy engine-base-branch: linrock/Stockfish/L1-128 num-epochs: 500 lambda: 1.0 ``` Experiment yaml configs converted to easy_train.sh commands with: https://github.com/linrock/nnue-tools/blob/4339954/yaml_easy_train.py Binpacks interleaved at training time with: official-stockfish/nnue-pytorch#259 Data filtered for high simple eval positions with: https://github.com/linrock/nnue-data/blob/32d6a68/filter_high_simple_eval_plain.py https://github.com/linrock/Stockfish/blob/61dbfe/src/tools/transform.cpp#L626-L655 Training data can be found at: https://robotmoon.com/nnue-training-data/ Local elo at 25k nodes per move of L1-128 smallnet (nnue-only eval) vs. L1-128 trained on standard S1 data: nn-epoch399.nnue : -318.1 +/- 2.1 Passed STC: https://tests.stockfishchess.org/tests/view/6574cb9d95ea6ba1fcd49e3b LLR: 2.93 (-2.94,2.94) <0.00,2.00> Total: 62432 W: 15875 L: 15521 D: 31036 Ptnml(0-2): 177, 7331, 15872, 7633, 203 Passed LTC: https://tests.stockfishchess.org/tests/view/6575da2d4d789acf40aaac6e LLR: 2.94 (-2.94,2.94) <0.50,2.50> Total: 64830 W: 16118 L: 15738 D: 32974 Ptnml(0-2): 43, 7129, 17697, 7497, 49 closes https://github.com/official-stockfish/Stockfish/pulls Bench: 1330050 Co-Authored-By: mstembera <5421953+mstembera@users.noreply.github.com>
official-stockfish · Jan 7, 2024 · 584d9ef · 584d9ef
1 parent a5a76a6
commit 584d9ef
Show file tree

Hide file tree

Showing 12 changed files with 297 additions and 198 deletions.
diff --git a/src/Makefile b/src/Makefile
@@ -806,7 +806,7 @@ help:
 	@echo "help                    > Display architecture details"
 	@echo "profile-build           > standard build with profile-guided optimization"
 	@echo "build                   > skip profile-guided optimization"
-	@echo "net                     > Download the default nnue net"
+	@echo "net                     > Download the default nnue nets"
 	@echo "strip                   > Strip executable"
 	@echo "install                 > Install executable"
 	@echo "clean                   > Clean up"
@@ -922,16 +922,7 @@ profileclean:
 	@rm -f stockfish.res
 	@rm -f ./-lstdc++.res
 
-# set up shell variables for the net stuff
-netvariables:
-	$(eval nnuenet := $(shell grep EvalFileDefaultName evaluate.h | grep define | sed 's/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/'))
-	$(eval nnuedownloadurl1 := https://tests.stockfishchess.org/api/nn/$(nnuenet))
-	$(eval nnuedownloadurl2 := https://github.com/official-stockfish/networks/raw/master/$(nnuenet))
-	$(eval curl_or_wget := $(shell if hash curl 2>/dev/null; then echo "curl -skL"; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi))
-	$(eval shasum_command := $(shell if hash shasum 2>/dev/null; then echo "shasum -a 256 "; elif hash sha256sum 2>/dev/null; then echo "sha256sum "; fi))
-
-# evaluation network (nnue)
-net: netvariables
+define fetch_network
 	@echo "Default net: $(nnuenet)"
 	@if [ "x$(curl_or_wget)" = "x" ]; then \
 		echo "Neither curl nor wget is installed. Install one of these tools unless the net has been downloaded manually"; \
@@ -966,7 +957,24 @@ net: netvariables
 		if [ "$(nnuenet)" = "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
 			echo "Network validated"; break; \
 		fi; \
-	fi; \
+	fi;
+endef
+
+# set up shell variables for the net stuff
+define netvariables
+$(eval nnuenet := $(shell grep $(1) evaluate.h | grep define | sed 's/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/'))
+$(eval nnuedownloadurl1 := https://tests.stockfishchess.org/api/nn/$(nnuenet))
+$(eval nnuedownloadurl2 := https://github.com/official-stockfish/networks/raw/master/$(nnuenet))
+$(eval curl_or_wget := $(shell if hash curl 2>/dev/null; then echo "curl -skL"; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi))
+$(eval shasum_command := $(shell if hash shasum 2>/dev/null; then echo "shasum -a 256 "; elif hash sha256sum 2>/dev/null; then echo "sha256sum "; fi))
+endef
+
+# evaluation network (nnue)
+net:
+	$(call netvariables, EvalFileDefaultNameBig)
+	$(call fetch_network)
+	$(call netvariables, EvalFileDefaultNameSmall)
+	$(call fetch_network)
 
 format:
 	$(CLANG-FORMAT) -i $(SRCS) $(HEADERS) -style=file

diff --git a/src/evaluate.cpp b/src/evaluate.cpp
@@ -23,6 +23,7 @@
 #include <cmath>
 #include <cstdlib>
 #include <fstream>
+#include <initializer_list>
 #include <iomanip>
 #include <iostream>
 #include <sstream>
@@ -31,6 +32,7 @@
 #include "incbin/incbin.h"
 #include "misc.h"
 #include "nnue/evaluate_nnue.h"
+#include "nnue/nnue_architecture.h"
 #include "position.h"
 #include "thread.h"
 #include "types.h"
@@ -44,19 +46,25 @@
 //     const unsigned int         gEmbeddedNNUESize;    // the size of the embedded file
 // Note that this does not work in Microsoft Visual Studio.
 #if !defined(_MSC_VER) && !defined(NNUE_EMBEDDING_OFF)
-INCBIN(EmbeddedNNUE, EvalFileDefaultName);
+INCBIN(EmbeddedNNUEBig, EvalFileDefaultNameBig);
+INCBIN(EmbeddedNNUESmall, EvalFileDefaultNameSmall);
 #else
-const unsigned char        gEmbeddedNNUEData[1] = {0x0};
-const unsigned char* const gEmbeddedNNUEEnd     = &gEmbeddedNNUEData[1];
-const unsigned int         gEmbeddedNNUESize    = 1;
+const unsigned char        gEmbeddedNNUEBigData[1]   = {0x0};
+const unsigned char* const gEmbeddedNNUEBigEnd       = &gEmbeddedNNUEBigData[1];
+const unsigned int         gEmbeddedNNUEBigSize      = 1;
+const unsigned char        gEmbeddedNNUESmallData[1] = {0x0};
+const unsigned char* const gEmbeddedNNUESmallEnd     = &gEmbeddedNNUESmallData[1];
+const unsigned int         gEmbeddedNNUESmallSize    = 1;
 #endif
 
 
 namespace Stockfish {
 
 namespace Eval {
 
-std::string currentEvalFileName = "None";
+std::string       currentEvalFileName[2] = {"None", "None"};
+const std::string EvFiles[2]             = {"EvalFile", "EvalFileSmall"};
+const std::string EvFileNames[2]         = {EvalFileDefaultNameBig, EvalFileDefaultNameSmall};
 
 // Tries to load a NNUE network at startup time, or when the engine
 // receives a UCI command "setoption name EvalFile value nn-[a-z0-9]{12}.nnue"
@@ -67,84 +75,96 @@ std::string currentEvalFileName = "None";
 // variable to have the engine search in a special directory in their distro.
 void NNUE::init() {
 
-    std::string eval_file = std::string(Options["EvalFile"]);
-    if (eval_file.empty())
-        eval_file = EvalFileDefaultName;
+    for (NetSize netSize : {Big, Small})
+    {
+        // change after fishtest supports EvalFileSmall
+        std::string eval_file =
+          std::string(netSize == Small ? EvalFileDefaultNameSmall : Options[EvFiles[netSize]]);
+        if (eval_file.empty())
+            eval_file = EvFileNames[netSize];
 
 #if defined(DEFAULT_NNUE_DIRECTORY)
-    std::vector<std::string> dirs = {"<internal>", "", CommandLine::binaryDirectory,
-                                     stringify(DEFAULT_NNUE_DIRECTORY)};
+        std::vector<std::string> dirs = {"<internal>", "", CommandLine::binaryDirectory,
+                                         stringify(DEFAULT_NNUE_DIRECTORY)};
 #else
-    std::vector<std::string> dirs = {"<internal>", "", CommandLine::binaryDirectory};
+        std::vector<std::string> dirs = {"<internal>", "", CommandLine::binaryDirectory};
 #endif
 
-    for (const std::string& directory : dirs)
-        if (currentEvalFileName != eval_file)
+        for (const std::string& directory : dirs)
         {
-            if (directory != "<internal>")
-            {
-                std::ifstream stream(directory + eval_file, std::ios::binary);
-                if (NNUE::load_eval(eval_file, stream))
-                    currentEvalFileName = eval_file;
-            }
-
-            if (directory == "<internal>" && eval_file == EvalFileDefaultName)
+            if (currentEvalFileName[netSize] != eval_file)
             {
-                // C++ way to prepare a buffer for a memory stream
-                class MemoryBuffer: public std::basic_streambuf<char> {
-                   public:
-                    MemoryBuffer(char* p, size_t n) {
-                        setg(p, p, p + n);
-                        setp(p, p + n);
-                    }
-                };
-
-                MemoryBuffer buffer(
-                  const_cast<char*>(reinterpret_cast<const char*>(gEmbeddedNNUEData)),
-                  size_t(gEmbeddedNNUESize));
-                (void) gEmbeddedNNUEEnd;  // Silence warning on unused variable
-
-                std::istream stream(&buffer);
-                if (NNUE::load_eval(eval_file, stream))
-                    currentEvalFileName = eval_file;
+                if (directory != "<internal>")
+                {
+                    std::ifstream stream(directory + eval_file, std::ios::binary);
+                    if (NNUE::load_eval(eval_file, stream, netSize))
+                        currentEvalFileName[netSize] = eval_file;
+                }
+
+                if (directory == "<internal>" && eval_file == EvFileNames[netSize])
+                {
+                    // C++ way to prepare a buffer for a memory stream
+                    class MemoryBuffer: public std::basic_streambuf<char> {
+                       public:
+                        MemoryBuffer(char* p, size_t n) {
+                            setg(p, p, p + n);
+                            setp(p, p + n);
+                        }
+                    };
+
+                    MemoryBuffer buffer(
+                      const_cast<char*>(reinterpret_cast<const char*>(
+                        netSize == Small ? gEmbeddedNNUESmallData : gEmbeddedNNUEBigData)),
+                      size_t(netSize == Small ? gEmbeddedNNUESmallSize : gEmbeddedNNUEBigSize));
+                    (void) gEmbeddedNNUEBigEnd;  // Silence warning on unused variable
+                    (void) gEmbeddedNNUESmallEnd;
+
+                    std::istream stream(&buffer);
+                    if (NNUE::load_eval(eval_file, stream, netSize))
+                        currentEvalFileName[netSize] = eval_file;
+                }
             }
         }
+    }
 }
 
 // Verifies that the last net used was loaded successfully
 void NNUE::verify() {
 
-    std::string eval_file = std::string(Options["EvalFile"]);
-    if (eval_file.empty())
-        eval_file = EvalFileDefaultName;
-
-    if (currentEvalFileName != eval_file)
+    for (NetSize netSize : {Big, Small})
     {
+        // change after fishtest supports EvalFileSmall
+        std::string eval_file =
+          std::string(netSize == Small ? EvalFileDefaultNameSmall : Options[EvFiles[netSize]]);
+        if (eval_file.empty())
+            eval_file = EvFileNames[netSize];
 
-        std::string msg1 =
-          "Network evaluation parameters compatible with the engine must be available.";
-        std::string msg2 = "The network file " + eval_file + " was not loaded successfully.";
-        std::string msg3 = "The UCI option EvalFile might need to specify the full path, "
-                           "including the directory name, to the network file.";
-        std::string msg4 = "The default net can be downloaded from: "
-                           "https://tests.stockfishchess.org/api/nn/"
-                         + std::string(EvalFileDefaultName);
-        std::string msg5 = "The engine will be terminated now.";
-
-        sync_cout << "info string ERROR: " << msg1 << sync_endl;
-        sync_cout << "info string ERROR: " << msg2 << sync_endl;
-        sync_cout << "info string ERROR: " << msg3 << sync_endl;
-        sync_cout << "info string ERROR: " << msg4 << sync_endl;
-        sync_cout << "info string ERROR: " << msg5 << sync_endl;
-
-        exit(EXIT_FAILURE);
-    }
+        if (currentEvalFileName[netSize] != eval_file)
+        {
+            std::string msg1 =
+              "Network evaluation parameters compatible with the engine must be available.";
+            std::string msg2 = "The network file " + eval_file + " was not loaded successfully.";
+            std::string msg3 = "The UCI option EvalFile might need to specify the full path, "
+                               "including the directory name, to the network file.";
+            std::string msg4 = "The default net can be downloaded from: "
+                               "https://tests.stockfishchess.org/api/nn/"
+                             + std::string(EvFileNames[netSize]);
+            std::string msg5 = "The engine will be terminated now.";
+
+            sync_cout << "info string ERROR: " << msg1 << sync_endl;
+            sync_cout << "info string ERROR: " << msg2 << sync_endl;
+            sync_cout << "info string ERROR: " << msg3 << sync_endl;
+            sync_cout << "info string ERROR: " << msg4 << sync_endl;
+            sync_cout << "info string ERROR: " << msg5 << sync_endl;
+
+            exit(EXIT_FAILURE);
+        }
 
-    sync_cout << "info string NNUE evaluation using " << eval_file << sync_endl;
+        sync_cout << "info string NNUE evaluation using " << eval_file << sync_endl;
+    }
 }
 }
 
-
 // Returns a static, purely materialistic evaluation of the position from
 // the point of view of the given color. It can be divided by PawnValue to get
 // an approximation of the material advantage on the board in terms of pawns.
@@ -163,18 +183,19 @@ Value Eval::evaluate(const Position& pos) {
     int   v;
     Color stm        = pos.side_to_move();
     int   shuffling  = pos.rule50_count();
-    int   simpleEval = simple_eval(pos, stm) + (int(pos.key() & 7) - 3);
-
-    bool lazy = std::abs(simpleEval) >= RookValue + KnightValue + 16 * shuffling * shuffling
-                                          + std::abs(pos.this_thread()->bestValue)
-                                          + std::abs(pos.this_thread()->rootSimpleEval);
+    int   simpleEval = simple_eval(pos, stm);
 
+    bool lazy = std::abs(simpleEval) > 2300;
     if (lazy)
         v = simpleEval;
     else
     {
-        int   nnueComplexity;
-        Value nnue = NNUE::evaluate(pos, true, &nnueComplexity);
+        bool smallNet = std::abs(simpleEval) > 1100;
+
+        int nnueComplexity;
+
+        Value nnue = smallNet ? NNUE::evaluate<NNUE::Small>(pos, true, &nnueComplexity)
+                              : NNUE::evaluate<NNUE::Big>(pos, true, &nnueComplexity);
 
         int optimism = pos.this_thread()->optimism[stm];
 
@@ -217,7 +238,7 @@ std::string Eval::trace(Position& pos) {
     ss << std::showpoint << std::showpos << std::fixed << std::setprecision(2) << std::setw(15);
 
     Value v;
-    v = NNUE::evaluate(pos, false);
+    v = NNUE::evaluate<NNUE::Big>(pos, false);
     v = pos.side_to_move() == WHITE ? v : -v;
     ss << "NNUE evaluation        " << 0.01 * UCI::to_cp(v) << " (white side)\n";
 

diff --git a/src/evaluate.h b/src/evaluate.h
@@ -34,12 +34,13 @@ std::string trace(Position& pos);
 int   simple_eval(const Position& pos, Color c);
 Value evaluate(const Position& pos);
 
-extern std::string currentEvalFileName;
+extern std::string currentEvalFileName[2];
 
 // The default net name MUST follow the format nn-[SHA256 first 12 digits].nnue
 // for the build process (profile-build and fishtest) to work. Do not change the
 // name of the macro, as it is used in the Makefile.
-#define EvalFileDefaultName "nn-b1e55edbea57.nnue"
+#define EvalFileDefaultNameBig "nn-b1e55edbea57.nnue"
+#define EvalFileDefaultNameSmall "nn-c01dc0ffeede.nnue"
 
 namespace NNUE {