Raise RuntimeError when canonicalization fails

The return value of xmlC14NExecute was not being checked, so
canonicalization failures (e.g., relative namespace URIs) silently
returned an empty string. This could allow downstream libraries to
accept invalid canonicalized output.

Check the return value and raise RuntimeError on failure, matching
JRuby's existing behavior.

This behavior was named as a contributing cause to GHSA-x4h9-gwv3-r4m4

Author: flavorjones

ext/nokogiri/xml_document.c

@@ -652,15 +652,19 @@ rb_xml_document_canonicalize(int argc, VALUE *argv, VALUE self)
     }
   }
 
-  xmlC14NExecute(c_doc, c_callback_wrapper, rb_callback,
-                 c_mode,
-                 c_namespaces,
-                 (int)RTEST(rb_comments_p),
-                 c_obuf);
+  int ret = xmlC14NExecute(c_doc, c_callback_wrapper, rb_callback,
+                           c_mode,
+                           c_namespaces,
+                           (int)RTEST(rb_comments_p),
+                           c_obuf);
 
   ruby_xfree(c_namespaces);
   xmlOutputBufferClose(c_obuf);
 
+  if (ret < 0) {
+    rb_raise(rb_eRuntimeError, "canonicalization failed");
+  }
+
   return rb_funcall(rb_io, rb_intern("string"), 0);
 }
 

test/xml/test_c14n.rb

@@ -199,6 +199,11 @@ def test_c14n_modes
         end
       end
 
+      def test_raise_on_canonicalization_failure
+        doc = Nokogiri.XML('<root xmlns:a="1"></root>')
+        assert_raises(RuntimeError) { doc.canonicalize }
+      end
+
       def test_wrong_params
         xml = "<a><b></b></a>"
         doc = Nokogiri.XML(xml)