Use libevhtp for outgoing HTTP requests.

Author: AlCutter

.travis.yml

@@ -21,6 +21,10 @@ before_install:
  - wget https://googlemock.googlecode.com/files/gmock-1.7.0.zip -O /tmp/gmock-1.7.0.zip
  - unzip -d /tmp /tmp/gmock-1.7.0.zip
  - sudo mv /tmp/gmock-1.7.0 /usr/src/gmock
+ - wget https://github.com/ellzey/libevhtp/archive/1.2.10.zip -O /tmp/libevhtp-1.2.10.zip
+ - unzip -d /tmp/ /tmp/libevhtp-1.2.10.zip
+ - mkdir /tmp/libevhtp-1.2.10/build
+ - pushd /tmp/libevhtp-1.2.10/build && cmake -DEVHTP_DISABLE_REGEX:STRING=ON .. && sudo make install; popd
  - sudo pip install -r python/requirements.txt
  - sudo ln -s protobuf-java.jar /usr/share/java/protobuf.jar
  - mkdir -p $GOPATH/src/github.com/google

Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get update && \
         libevent-2.0.5 \
         libevent-core-2.0.5 \
         libevent-extra-2.0.5 \
+        libevent-openssl-2.0.5 \
         libevent-pthreads-2.0.5 \
         libgflags2 \
         libgoogle-glog0 \

Dockerfile-ct-mirror

@@ -10,6 +10,7 @@ RUN apt-get update && \
         libevent-2.0.5 \
         libevent-core-2.0.5 \
         libevent-extra-2.0.5 \
+        libevent-openssl-2.0.5 \
         libevent-pthreads-2.0.5 \
         libgflags2 \
         libgoogle-glog0 \
@@ -51,7 +52,7 @@ CMD cd /mnt/ctmirror/ && \
       CORE_DIR="/mnt/ctmirror/cores/$(date +%s)"; \
       mkdir -p ${CORE_DIR}; \
       cp -v core ${CORE_DIR}; \
-      cp -v /usr/local/bin/ct-server ${CORE_DIR}; \
+      cp -v /usr/local/bin/ct-mirror ${CORE_DIR}; \
       echo "Core saved to ${CORE_DIR}"; \
     fi
 

Makefile.am

@@ -163,7 +163,7 @@ cpp_server_ct_mirror_LDADD = \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
 	$(leveldb_LIBS) \
-	-lcrypto -lprotobuf -lsqlite3
+	-lcrypto -lssl -lprotobuf -lsqlite3 -levhtp
 cpp_server_ct_mirror_SOURCES = \
 	cpp/client/async_log_client.cc \
 	cpp/fetcher/remote_peer.cc \
@@ -189,7 +189,7 @@ cpp_server_ct_server_LDADD = \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
 	$(leveldb_LIBS) \
-	-lcrypto -lprotobuf -lsqlite3
+	-lcrypto -lssl -lprotobuf -lsqlite3 -levhtp
 cpp_server_ct_server_SOURCES = \
 	cpp/client/async_log_client.cc \
 	cpp/proto/serializer.cc \
@@ -213,7 +213,7 @@ cpp_tools_ct_clustertool_LDADD = \
 	cpp/libcore.a \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
-	-lprotobuf -lcrypto -lsqlite3
+	-lprotobuf -lcrypto -lssl -lsqlite3 -levhtp
 cpp_tools_ct_clustertool_SOURCES = \
 	cpp/proto/serializer.cc \
 	cpp/tools/clustertool_main.cc \
@@ -229,7 +229,7 @@ cpp_client_ct_LDADD = \
 	cpp/libcore.a \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
-	-lprotobuf -lcrypto -lssl -lsqlite3
+	-lprotobuf -lcrypto -lssl -lsqlite3 -levhtp
 cpp_client_ct_SOURCES = \
 	cpp/client/async_log_client.cc \
 	cpp/client/client.cc \
@@ -272,7 +272,8 @@ cpp_tools_dump_sth_SOURCES = \
 cpp_tools_etcd_watch_LDADD = \
 	cpp/libcore.a \
 	$(json_c_LIBS) \
-	$(libevent_LIBS)
+	$(libevent_LIBS) \
+	-lcrypto -lssl -levhtp
 cpp_tools_etcd_watch_SOURCES = \
 	cpp/tools/etcd_watch.cc \
 	cpp/util/json_wrapper.cc \
@@ -281,7 +282,8 @@ cpp_tools_etcd_watch_SOURCES = \
 cpp_util_bench_etcd_LDADD = \
 	cpp/libcore.a \
 	$(json_c_LIBS) \
-	$(libevent_LIBS)
+	$(libevent_LIBS) \
+	-lcrypto -lssl -levhtp
 cpp_util_bench_etcd_SOURCES = \
 	cpp/util/bench_etcd.cc \
 	cpp/util/json_wrapper.cc \
@@ -291,7 +293,7 @@ cpp_util_etcd_masterelection_LDADD = \
 	cpp/libcore.a \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
-	-lprotobuf
+	-lprotobuf -lcrypto -lssl -levhtp
 cpp_util_etcd_masterelection_SOURCES = \
 	cpp/util/etcd_masterelection.cc \
 	cpp/util/json_wrapper.cc \
@@ -312,7 +314,7 @@ cpp_log_cluster_state_controller_test_LDADD = \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
 	$(leveldb_LIBS) \
-	-lprotobuf -lsqlite3 -lcrypto
+	-lprotobuf -lsqlite3 -lcrypto -lssl -levhtp
 cpp_log_cluster_state_controller_test_SOURCES = \
 	cpp/client/async_log_client.cc \
 	cpp/log/cluster_state_controller_test.cc \
@@ -343,7 +345,7 @@ cpp_log_etcd_consistent_store_test_LDADD = \
 	cpp/libtest.a \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
-	-lprotobuf -lcrypto
+	-lprotobuf -lcrypto -lssl -levhtp
 cpp_log_etcd_consistent_store_test_SOURCES = \
 	cpp/log/etcd_consistent_store_test.cc \
 	cpp/proto/serializer.cc \
@@ -373,7 +375,7 @@ cpp_log_frontend_signer_test_LDADD = \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
 	$(leveldb_LIBS) \
-	-lprotobuf -lcrypto -lsqlite3
+	-lprotobuf -lcrypto -lssl -lsqlite3 -levhtp
 cpp_log_frontend_signer_test_SOURCES = \
 	cpp/log/frontend_signer_test.cc \
 	cpp/log/test_signer.cc \
@@ -391,7 +393,7 @@ cpp_log_log_lookup_test_LDADD = \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
 	$(leveldb_LIBS) \
-	-lprotobuf -lcrypto -lsqlite3
+	-lprotobuf -lcrypto -lssl -lsqlite3 -levhtp
 cpp_log_log_lookup_test_SOURCES = \
 	cpp/log/log_lookup_test.cc \
 	cpp/log/test_signer.cc \
@@ -429,7 +431,7 @@ cpp_log_strict_consistent_store_test_LDADD = \
 	cpp/libtest.a \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
-	-lprotobuf
+	-lprotobuf -lcrypto -lssl -levhtp
 cpp_log_strict_consistent_store_test_SOURCES = \
 	cpp/log/strict_consistent_store_cert.cc \
 	cpp/log/strict_consistent_store_test.cc \
@@ -444,7 +446,7 @@ cpp_log_tree_signer_test_LDADD = \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
 	$(leveldb_LIBS) \
-	-lprotobuf -lcrypto -lsqlite3
+	-lprotobuf -lcrypto -lssl -lsqlite3 -levhtp
 cpp_log_tree_signer_test_SOURCES = \
 	cpp/log/test_signer.cc \
 	cpp/log/tree_signer_test.cc \
@@ -522,7 +524,7 @@ cpp_server_proxy_test_LDADD = \
 	cpp/libtest.a \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
-	-lprotobuf
+	-lprotobuf -lcrypto -lssl -levhtp
 cpp_server_proxy_test_SOURCES = \
 	cpp/server/json_output.cc \
 	cpp/server/proxy.cc \
@@ -535,7 +537,8 @@ cpp_util_etcd_delete_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
 	$(json_c_LIBS) \
-	$(libevent_LIBS)
+	$(libevent_LIBS) \
+	-lcrypto -lssl -levhtp
 cpp_util_etcd_delete_test_SOURCES = \
 	cpp/util/etcd_delete_test.cc \
 	cpp/util/json_wrapper.cc \
@@ -546,7 +549,8 @@ cpp_util_etcd_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
 	$(json_c_LIBS) \
-	$(libevent_LIBS)
+	$(libevent_LIBS) \
+	-lcrypto -lssl -levhtp
 cpp_util_etcd_test_SOURCES = \
 	cpp/util/etcd_test.cc \
 	cpp/util/json_wrapper.cc \
@@ -556,7 +560,8 @@ cpp_util_fake_etcd_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
 	$(json_c_LIBS) \
-	$(libevent_LIBS)
+	$(libevent_LIBS) \
+	-lcrypto -lssl -levhtp
 cpp_util_fake_etcd_test_SOURCES = \
 	cpp/util/fake_etcd_test.cc \
 	cpp/util/json_wrapper.cc \
@@ -575,7 +580,8 @@ cpp_util_json_wrapper_test_SOURCES = \
 cpp_util_libevent_wrapper_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
-	$(libevent_LIBS)
+	$(libevent_LIBS) \
+	-lcrypto -lssl -levhtp
 cpp_util_libevent_wrapper_test_SOURCES = \
 	cpp/util/libevent_wrapper.cc \
 	cpp/util/libevent_wrapper_test.cc
@@ -586,7 +592,7 @@ cpp_util_masterelection_test_LDADD = \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
 	$(leveldb_LIBS) \
-	-lprotobuf
+	-lprotobuf -lcrypto -lssl -levhtp
 cpp_util_masterelection_test_SOURCES = \
 	cpp/util/json_wrapper.cc \
 	cpp/util/libevent_wrapper.cc \
@@ -598,7 +604,7 @@ cpp_merkletree_merkle_tree_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
 	$(libevent_LIBS) \
-	-lcrypto
+	-lcrypto -lssl -levhtp
 cpp_merkletree_merkle_tree_test_SOURCES = \
 	cpp/util/util.cc \
 	cpp/merkletree/merkle_tree_test.cc
@@ -607,7 +613,7 @@ cpp_merkletree_serial_hasher_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
 	$(libevent_LIBS) \
-	-lcrypto
+	-lcrypto -lssl -levhtp
 cpp_merkletree_serial_hasher_test_SOURCES = \
 	cpp/util/util.cc \
 	cpp/merkletree/serial_hasher_test.cc
@@ -616,31 +622,34 @@ cpp_merkletree_tree_hasher_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
 	$(libevent_LIBS) \
-	-lcrypto
+	-lcrypto -lssl -levhtp
 cpp_merkletree_tree_hasher_test_SOURCES = \
 	cpp/util/util.cc \
 	cpp/merkletree/tree_hasher_test.cc
 
 cpp_util_sync_task_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
-	$(libevent_LIBS)
+	$(libevent_LIBS) \
+	-lcrypto -lssl -levhtp
 cpp_util_sync_task_test_SOURCES = \
 	cpp/util/sync_task_test.cc \
 	cpp/util/thread_pool.cc
 
 cpp_util_task_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
-	$(libevent_LIBS)
+	$(libevent_LIBS) \
+	-lcrypto -lssl -levhtp
 cpp_util_task_test_SOURCES = \
 	cpp/util/task_test.cc \
 	cpp/util/thread_pool.cc
 
 cpp_util_thread_pool_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
-	$(libevent_LIBS)
+	$(libevent_LIBS) \
+	-lcrypto -lssl -levhtp
 cpp_util_thread_pool_test_SOURCES = \
 	cpp/util/thread_pool_test.cc \
 	cpp/util/thread_pool.cc
@@ -649,7 +658,7 @@ cpp_log_cert_checker_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
 	$(libevent_LIBS) \
-	-lcrypto
+	-lcrypto -lssl -levhtp
 cpp_log_cert_checker_test_SOURCES = \
 	cpp/log/cert_checker_test.cc \
 	cpp/util/openssl_util.cc \
@@ -659,7 +668,7 @@ cpp_log_cert_submission_handler_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
 	$(libevent_LIBS) \
-	-lprotobuf -lcrypto
+	-lprotobuf -lcrypto -lssl -levhtp
 cpp_log_cert_submission_handler_test_SOURCES = \
 	cpp/log/cert_submission_handler_test.cc \
 	cpp/util/openssl_util.cc \
@@ -669,7 +678,7 @@ cpp_log_cert_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
 	$(libevent_LIBS) \
-	-lcrypto
+	-lcrypto -lssl -levhtp
 cpp_log_cert_test_SOURCES = \
 	cpp/log/cert_test.cc \
 	cpp/util/openssl_util.cc \
@@ -679,7 +688,7 @@ cpp_log_ct_extensions_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
 	$(libevent_LIBS) \
-	-lcrypto
+	-lcrypto -lssl -levhtp
 cpp_log_ct_extensions_test_SOURCES = \
 	cpp/log/ct_extensions_test.cc \
 	cpp/util/openssl_util.cc \
@@ -690,7 +699,7 @@ cpp_log_database_large_test_LDADD = \
 	cpp/libtest.a \
 	$(libevent_LIBS) \
 	$(leveldb_LIBS) \
-	-lprotobuf -lcrypto -lsqlite3
+	-lprotobuf -lcrypto -lssl -lsqlite3 -levhtp
 cpp_log_database_large_test_SOURCES = \
 	cpp/log/database_large_test.cc \
 	cpp/log/test_signer.cc \
@@ -704,7 +713,7 @@ cpp_log_frontend_test_LDADD = \
 	$(json_c_LIBS) \
 	$(libevent_LIBS) \
 	$(leveldb_LIBS) \
-	-lprotobuf -lcrypto -lsqlite3
+	-lprotobuf -lcrypto -lssl -lsqlite3 -levhtp
 cpp_log_frontend_test_SOURCES = \
 	cpp/log/frontend_test.cc \
 	cpp/log/test_signer.cc \
@@ -721,7 +730,7 @@ cpp_merkletree_merkle_tree_large_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \
 	$(libevent_LIBS) \
-	-lcrypto
+	-lcrypto -lssl -levhtp
 cpp_merkletree_merkle_tree_large_test_SOURCES = \
 	cpp/merkletree/merkle_tree_large_test.cc \
 	cpp/util/util.cc

README.md

@@ -54,6 +54,9 @@ installing. If you can install the missing header manually, it should
 work.
 
  - [libevent](http://libevent.org/) (tested with 2.0.21-stable)
+ - [libevhtp](https://github.com/ellzey/libevhtp) (tested with 1.2.10)
+ If building libevhtp from source, you may need to disable the regex support
+ with the following cmake flag: `-DEVHTP_DISABLE_REGEX:STRING=ON`
 
 You can specify a non-installed locally built library using the
 `LIBEVENTDIR` environment variable to point to the local build. Note

configure.ac

@@ -38,6 +38,8 @@ AC_CHECK_HEADER([google/protobuf/message.h],,
                 [AC_MSG_ERROR([protobuf headers could not be found])])
 AC_CHECK_HEADER([leveldb/db.h],,
                 [AC_MSG_ERROR([leveldb headers could not be found])])
+AC_CHECK_HEADER([evhtp.h],,
+                [AC_MSG_ERROR([libevhtp headers could not be found])])
 AC_CHECK_HEADER([ldns/ldns.h],, [missing_ldns=yes])
 
 # Check for working GTest/GMock.
@@ -61,6 +63,7 @@ AS_IF([test -n "$missing_gmock"],
 # Checks for libraries.
 AC_SEARCH_LIBS([__b64_ntop], [resolv])
 AC_SEARCH_LIBS([pthread_create], [pthread])
+AC_SEARCH_LIBS([evhtp_make_request], [evhtp])
 
 AC_MSG_CHECKING([checking for gflags library])
 LIBS="-lgflags $LIBS"
@@ -91,9 +94,13 @@ AC_SEARCH_LIBS([event_base_dispatch], [event],, [missing_libevent=1],
                [$save_LIBS])
 AC_SEARCH_LIBS([evthread_use_pthreads], [event_pthreads],, [missing_libevent=1],
                [$save_LIBS])
+AC_SEARCH_LIBS([bufferevent_openssl_socket_new], [event_openssl],, [missing_libevent_openssl=1],
+               [$save_LIBS])
 AC_SUBST([libevent_LIBS], [$LIBS])
 AS_IF([test -n "$missing_libevent"],
       [AC_MSG_ERROR([could not find the libevent libraries])])
+AS_IF([test -n "$missing_libevent_openssl"],
+      [AC_MSG_ERROR([could not find the libevent_openssl library])])
 LIBS="$save_LIBS"
 
 # TCMalloc gubbins