diff --git a/nongnu/packages/clojure.scm b/nongnu/packages/clojure.scm
index 25170ed..234fafd 100644
--- a/nongnu/packages/clojure.scm
+++ b/nongnu/packages/clojure.scm
@@ -101,7 +101,6 @@ lets you focus on your code.")
     (list #:patchelf-plan `'(("clj-kondo" ("gcc" "zlib")))
           #:install-plan `'(("clj-kondo" "/bin/"))
           #:phases #~(modify-phases %standard-phases
-                       (delete 'binary-unpack)
                        (add-after 'unpack 'chmod
                          (lambda _
                            (chmod "clj-kondo" #o755))))))
diff --git a/nongnu/packages/game-development.scm b/nongnu/packages/game-development.scm
index 0e28127..566cea2 100644
--- a/nongnu/packages/game-development.scm
+++ b/nongnu/packages/game-development.scm
@@ -144,7 +144,6 @@ development should opt for GLSL rather than Cg.")
              (copy-file (assoc-ref inputs "source") "libsteam_api.so")
              (chmod "libsteam_api.so" #o644)
              #t))
-         (delete 'binary-unpack)
          (add-after 'install 'symlink
            (lambda* (#:key outputs #:allow-other-keys)
              (let ((out (assoc-ref outputs "out")))
diff --git a/nonguix/build/binary-build-system.scm b/nonguix/build/binary-build-system.scm
index 147aa18..e5bbc48 100644
--- a/nonguix/build/binary-build-system.scm
+++ b/nonguix/build/binary-build-system.scm
@@ -155,7 +155,12 @@ The inputs are optional when the file is an executable."
       (chdir "binary")
       (match binary-file
         ((? deb-file?) (unpack-deb binary-file))
-        (_ (format #t "Unknown file type: ~a~%" (basename binary-file)))))))
+        (_
+         (begin
+           (format #t "Unknown file type: ~a~%" (basename binary-file))
+           ;; Cleanup after ourselves
+           (chdir "..")
+           (rmdir "binary")))))))
 
 (define %standard-phases
   ;; Everything is as with the GNU Build System except for the `binary-unpack',