diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 09c8d9b3b5..64d92faa76 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,6 +12,16 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest,macos-latest]
+        # without-x:
+        # - false: automatically configure X11
+        # - true: disable building with X11 (use --without-x option)
+        without-x: [false,true]
+        # used for macos runner
+        homebrew-unlink: [ false ]
+        include:
+          - os: macos-latest
+            without-x: false
+            homebrew-unlink: true
     steps:
       - uses: actions/checkout@v2
       - name: Pre-reqs (apt)
@@ -26,17 +36,31 @@ jobs:
           brew install automake \
             libpng \
             libxt
+      - name: Prepare for linking with Homebrew
+        if: runner.os == 'macOS'
+        run: |
+          if ! ${{ matrix.homebrew-unlink }}; then
+            echo "Using Homebrew prefix /usr/local/lib for linking via LIBRARY_PATH in order to keep the default search path."
+            echo "LIBRARY_PATH=$(brew --prefix)/lib${LIBRARY_PATH:+:${LIBRARY_PATH}}" >> $GITHUB_ENV
+          else
+            echo "Unlinking libraries from Homebrew prefix so that pkg-config must be used."
+            FORMULAE_TO_UNLINK="libpng libxt libice libsm libx11"
+            brew unlink $FORMULAE_TO_UNLINK
+            PKG_CONFIG_PATH_FOR_FORMULAE=$( brew ls $FORMULAE_TO_UNLINK \
+              | grep /pkgconfig/ | xargs -n1 dirname \
+              | sort -u | tr '\n' ':' | sed 's/:$//' )
+            echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH_FOR_FORMULAE${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" >> $GITHUB_ENV
+          fi
       - name: Bootstrap
         run: |
           sh ./bootstrap
       - name: Configure
-        run: |
+        run:
           sh ./configure
+            ${{ fromJSON('[ "", "--without-x" ]')[ matrix.without-x ] }}
       - name: Make
-        if: runner.os == 'Linux'
         run: |
           make
-      - name: Make
-        if: runner.os == 'macOS'
+      - name: Show pkg-config files
         run: |
-          make LDFLAGS="$(pkg-config --libs --static libpng xt)"
+          head -n100 *.pc
diff --git a/.gitignore b/.gitignore
index 935b0466c6..622a38c859 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@ pgplot_link_adam
 .libs
 *.la
 *.lo
+*.o
 Makefile
 Makefile.in
 acinclude.m4
diff --git a/configure.ac b/configure.ac
index 801d24d610..aefab6b9f3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -73,6 +73,60 @@ dnl   automatically (see the automake manual's FAQ for justification).
 drivers="exdriv.lo pgdriv.lo cadriv.lo hgdriv.lo lxdriv.lo  vtdriv.lo hidriv.lo psdriv.lo ttdriv.lo cwdriv.lo gldriv.lo nedriv.lo hpdriv.lo lsdriv.lo nudriv.lo qmdriv.lo vadriv.lo cgdriv.lo nexsup.lo rvdriv.lo xmdriv.lo tkdriv.lo pkdriv.lo xadriv.lo"
 
 dnl   Not all platforms have X
+have_x11=no
+have_xt=no
+dnl	+--------+------------------------+----------+----------+--------+-----+
+dnl	|Variable|      AC_PATH_XTRA      |-without-x|-with-x=no|-with-x | --x |
+dnl	+--------+------------+-----------+----------+----------+--------+-----+
+dnl	|        |  success   |  failure  |          |          |        |     |
+dnl	+--------+------------+-----------+----------+----------+--------+-----+
+dnl	|  no_x  |     ''     |   'yes'   |  'yes'   |          |        |     |
+dnl	+--------+------------+-----------+----------+----------+--------+-----+
+dnl	| with_x |            |           |   'no'   |   'no'   | 'yes'  |'yes'|
+dnl	+--------+------------+-----------+----------+----------+--------+-----+
+dnl	Notes:
+dnl     1) AC_PATH_XTRA runs first and modifies no_x in the presence of -without-x
+dnl     2) If --with-x is not specified, AC_PATH_XTRA assumes with_x='yes'
+dnl
+dnl     Check pkg-config if the user did not forbid X and AC_PATH_XTRA failed to find it
+AS_IF([ test "x$with_x" != xno -a "x$no_x" != x ],
+
+      [
+       PKG_CHECK_MODULES(
+          [X11],
+          [x11],
+          [have_x11=yes
+           no_x=""
+           PKG_CONFIG_REQUIRES="${PKG_CONFIG_REQUIRES}[,]x11"
+           PKG_CHECK_MODULES(
+              [XT],
+              dnl have sm and ice listed here so that contents of $X_PRE_LIBS
+              dnl can be found
+              [sm ice xt],
+              [have_xt=yes
+               PKG_CONFIG_REQUIRES="${PKG_CONFIG_REQUIRES}[,]xt"
+               dnl XT_* has xt + x11 because xt depends on x11
+               X_LIBS="$X_LIBS $XT_LIBS"
+               LIBS="$LIBS $XT_LIBS"
+               X_CFLAGS="$X_CFLAGS $XT_CFLAGS"
+               CPPFLAGS="$CPPFLAGS $XT_CFLAGS"
+              ],
+              [
+               dnl only have x11
+               X_LIBS="$X_LIBS $X11_LIBS"
+               LIBS="$LIBS $X11_LIBS"
+               X_CFLAGS="$X_CFLAGS $X11_CFLAGS"
+               CPPFLAGS="$CPPFLAGS $X11_CFLAGS"
+              ]
+           )
+          ],
+          [
+           dnl no x11 via pkg-config either
+          ]
+       )
+      ]
+)
+
 COMPILE_PGDISP=no
 AS_IF([ test "x$no_x" = "x"],
       [