From bbad9f255d76cf7dcd11d15800e508d9849826af Mon Sep 17 00:00:00 2001 From: Nelson Billing Date: Wed, 19 Feb 2020 15:32:08 -0800 Subject: [PATCH] Add optional new symbol upload API to sym_upload. Change-Id: I6a49e9f4a699fa6f5f8e9f0fc86afb4cb342a442 Reviewed-on: https://chromium-review.googlesource.com/c/breakpad/breakpad/+/1422400 Reviewed-by: Mark Mentovai Reviewed-by: Ivan Penkov Reviewed-by: Mike Frysinger --- Makefile.am | 4 + Makefile.in | 14 ++ docs/sym_upload_v2_protocol.md | 214 ++++++++++++++++++ src/common/common.gyp | 6 +- src/common/linux/libcurl_wrapper.cc | 227 ++++++++++++++------ src/common/linux/libcurl_wrapper.h | 28 ++- src/common/linux/symbol_collector_client.cc | 193 +++++++++++++++++ src/common/linux/symbol_collector_client.h | 87 ++++++++ src/common/linux/symbol_upload.cc | 151 +++++++++++-- src/common/linux/symbol_upload.h | 14 +- src/tools/linux/symupload/sym_upload.cc | 41 +++- 11 files changed, 880 insertions(+), 99 deletions(-) create mode 100644 docs/sym_upload_v2_protocol.md create mode 100644 src/common/linux/symbol_collector_client.cc create mode 100644 src/common/linux/symbol_collector_client.h diff --git a/Makefile.am b/Makefile.am index 3072a473..d5f60a3f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -615,6 +615,10 @@ src_tools_linux_symupload_minidump_upload_LDADD = -ldl src_tools_linux_symupload_sym_upload_SOURCES = \ src/common/linux/http_upload.cc \ src/common/linux/http_upload.h \ + src/common/linux/libcurl_wrapper.cc \ + src/common/linux/libcurl_wrapper.h \ + src/common/linux/symbol_collector_client.cc \ + src/common/linux/symbol_collector_client.h \ src/common/linux/symbol_upload.cc \ src/common/linux/symbol_upload.h \ src/tools/linux/symupload/sym_upload.cc diff --git a/Makefile.in b/Makefile.in index 4b23a877..96ad72eb 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1495,10 +1495,16 @@ src_tools_linux_symupload_minidump_upload_OBJECTS = \ src_tools_linux_symupload_minidump_upload_DEPENDENCIES = am__src_tools_linux_symupload_sym_upload_SOURCES_DIST = \ src/common/linux/http_upload.cc src/common/linux/http_upload.h \ + src/common/linux/libcurl_wrapper.cc \ + src/common/linux/libcurl_wrapper.h \ + src/common/linux/symbol_collector_client.cc \ + src/common/linux/symbol_collector_client.h \ src/common/linux/symbol_upload.cc \ src/common/linux/symbol_upload.h \ src/tools/linux/symupload/sym_upload.cc @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@am_src_tools_linux_symupload_sym_upload_OBJECTS = src/common/linux/http_upload.$(OBJEXT) \ +@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.$(OBJEXT) \ +@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.$(OBJEXT) \ @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.$(OBJEXT) \ @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/tools/linux/symupload/sym_upload.$(OBJEXT) src_tools_linux_symupload_sym_upload_OBJECTS = \ @@ -2477,6 +2483,10 @@ TESTS = $(check_PROGRAMS) $(check_SCRIPTS) @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@src_tools_linux_symupload_sym_upload_SOURCES = \ @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/http_upload.cc \ @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/http_upload.h \ +@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.cc \ +@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.h \ +@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.cc \ +@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.h \ @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.cc \ @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.h \ @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/tools/linux/symupload/sym_upload.cc @@ -4726,6 +4736,9 @@ src/tools/linux/symupload/minidump_upload.$(OBJEXT): \ src/tools/linux/symupload/minidump_upload$(EXEEXT): $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_DEPENDENCIES) $(EXTRA_src_tools_linux_symupload_minidump_upload_DEPENDENCIES) src/tools/linux/symupload/$(am__dirstamp) @rm -f src/tools/linux/symupload/minidump_upload$(EXEEXT) $(AM_V_CXXLD)$(CXXLINK) $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_LDADD) $(LIBS) +src/common/linux/symbol_collector_client.$(OBJEXT): \ + src/common/linux/$(am__dirstamp) \ + src/common/linux/$(DEPDIR)/$(am__dirstamp) src/common/linux/symbol_upload.$(OBJEXT): \ src/common/linux/$(am__dirstamp) \ src/common/linux/$(DEPDIR)/$(am__dirstamp) @@ -4996,6 +5009,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-linux_libc_support.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-memory_mapped_file.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-safe_readlink.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/symbol_collector_client.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/symbol_upload.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_client_linux_linux_client_unittest_shlib-crash_generator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_common_dumper_unittest-crash_generator.Po@am__quote@ diff --git a/docs/sym_upload_v2_protocol.md b/docs/sym_upload_v2_protocol.md new file mode 100644 index 00000000..d56a4c83 --- /dev/null +++ b/docs/sym_upload_v2_protocol.md @@ -0,0 +1,214 @@ +# Introduction + +The `sym_upload` tool is able to operate in `sym-upload-v2` protocol mode, in +addition to the legacy protocol (which will be referred to as `sym-upload-v1` +for the rest of this document). For now `sym-upload-v2` is HTTP/REST-based but +it could be extended to operate over gRPC instead, in the future. + +# Table of Contents +* [Why](#why) +* [How](#how) + * [Uploading](#uploading) + * [Uploading with `sym_upload`](#uploading-with-sym_upload) + * [Uploading with curl](#uploading-with-curl) + * [Serving the `sym-upload-v2` protocol](#serving-the-sym-upload-v2-protocol) + * [Authenticate using `key`](#authenticate-using-key) + * [Symbol `checkStatus`](#symbol-checkstatus) + * [Upload `create`](#upload-create) + * [Uploading the symbol file](#uploading-the-symbol-file) + * [Upload complete](#upload-complete) + + +# Why + +Using `sym_upload` in `sym-upload-v2` protocol mode has the following features +beyond `sym-upload-v1`: + * Authentication via `key` (arbitrary secret). + * Symbol identifier (product of `debug_file` and `debug_id`, as recorded in +output from `dump_syms`) can be checked against existing symbol information on +server. If it's present, then the upload is skipped entirely. + +# How + +## Uploading + +### Uploading with `sym_upload` + +Uploading in `sym-upload-v2` protocol mode is easy. Invoke `sym_upload` like +``` +$ ./sym_upload -p sym-upload-v2 [-k ] +``` + +Where `symbol-file` is a symbol file created by `dump_syms`, `API-URL` is the +URL of your `sym-upload-v2` API service (see next section for details), and +`API-key` is a secret known to your uploader and server. + +For more options see `sym_upload --help`. + +### Uploading with curl + +As an example, if: + * Your API's URL was "https://sym-upload-api". + * Your service has assigned you `key` "myfancysecret123". + * You wanted to upload the symbol file at "path/to/file_name", with +`debug_file` being "file_name" and `debug_id` being +"123123123123123123123123123". Normally you would read these values from +"path/to/file_name", which in turn was generated by `dump_syms`. + +Then you might run: +``` +$ curl https://sym-upload-api/symbols/file_name/123123123123123123123123123:checkStatus?key=myfancysecret123 +``` + +And, upon seeing that this `debug_file`/`debug_id` combo is missing from symbol +storage then you could run: +``` +$ curl --request POST https://sym-upload-api/uploads:create?key=myfancysecret123 +``` + +Which returns `upload_url` "https://upload-server/42?creds=shhhhh" and +`upload_key` "42". Next you upload the file directly like: +``` +$ curl -T path/to/file_name "https://upload-server/42?creds=shhhhh" +``` + +Once the HTTP PUT is complete, run: +``` +$ curl --header "Content-Type: application/json" \ + --request POST \ + --data '{symbol_id:{"debugFile":"file_name",'\ + '"debugId":"123123123123123123123123123"}}' \ + https://sym-upload-api/uploads/42:complete?key=myfancysecret123 +``` + +### Serving the `sym-upload-v2` Protocol + +The protocol is currently defined only in HTTP/REST. There are three necessary +REST operations to implement in your service: +* `/symbols//:checkStatus?key=` +* `/uploads:create?key=` +* `/uploads/:complete?key=` + +#### Authenticate Using `key` + +The query string arg `key` contains some secret that both the uploader and +server understand. It is up to the service implementer to decide on what +constitutes a valid `key`, how the uploader acquires one, and how to handle +requests made with invalid ones. + +#### Symbol `checkStatus` + +``` +/symbols//:checkStatus?key= +``` + +This operation expects an empty (or no) JSON payload in the request. + +This operation should return the status of the symbol file uniquely identified +by the given `debug_file` and `debug_id`. JSON schema: +``` +{ + "type": object", + "properties": { + "status": { + "type": "string", + "enum": ["STATUS_UNSPECIFIED", "MISING", "FOUND"], + "required": true + } + } +} +``` + +Where `MISSING` denotes that the symbol file does not exist on the server and +`FOUND` denotes that the symbol file exists on the server. + +#### Upload `create` + +``` +/uploads:create?key= +``` + +This operation expects an empty (or no) JSON payload in the request. + +This operation should return a URL that uploader can HTTP PUT their symbol file +to, along with an "upload key" that can be used to notify the service once the +file upload is completed. JSON schema: +``` +{ + "type": "object", + "properties": { + "upload_url": { + "type: "string", + "required": true + }, + "upload_key": { + "type": "string", + "required": true + } + } +} +``` + +Since this REST API operation can be authenticated via the `key` query string +arg, the service can return a URL that encodes permission delegation to the +upload endpoint resource and thereby constrain the ability to upload to those +with valid `key`s. + +#### Uploading the Symbol File + +Note that the actual symbol upload step is _not_ part of the REST API. The +upload URL obtained in the above operation is meant to be used as the endpoint +for a normal HTTP PUT request for the contents of the symbol file. Once that +HTTP PUT request is completed use the upload `complete` operation. + +#### Upload `complete` + +``` +/uploads/:complete?key= +``` + +This operation expects a JSON payload in the HTTP request body with the +following schema: +``` +{ + "type": "object", + "properties": { + "symbol_id": { + "type": "object", + "properties": { + "debug_file": { + "type": "string", + "required": true + }, + "debug_id": { + "type": "string", + "required": true + } + } + } + } +} +``` + +This operation should cause the symbol storage back-end (however implemented) +to consume the symbol file identified by `upload_key`. It is up to the service +implementation to decide how uploads are assigned `upload_key`s and how to +retrieve a completed upload by its `upload_key`. If the symbol file cannot be +found, is malformed, or the operation cannot be completed for any other reason +then an HTTP error will be returned. JSON schema of non-error responses: +``` +{ + "type": "object", + "properties": { + "result": { + "type": string, + "enum": ["RESULT_UNSPECIFIED", "OK", "DUPLICATE_DATA"], + "required": true + } + } +} +``` + +Where `OK` denotes that the symbol storage was updated with the new symbol file +and `DUPLICATE_DATA` denotes that the symbol file data was identical to data +already in symbol storage and therefore nothing changed. diff --git a/src/common/common.gyp b/src/common/common.gyp index 7d5e5c7d..c0d71a3a 100644 --- a/src/common/common.gyp +++ b/src/common/common.gyp @@ -74,8 +74,8 @@ 'dwarf/dwarf2reader.cc', 'dwarf/dwarf2reader.h', 'dwarf/dwarf2reader_test_common.h', - 'dwarf/elf_reader.cc', - 'dwarf/elf_reader.h', + 'dwarf/elf_reader.cc', + 'dwarf/elf_reader.h', 'dwarf/functioninfo.cc', 'dwarf/functioninfo.h', 'dwarf/line_state_machine.h', @@ -118,6 +118,8 @@ 'linux/memory_mapped_file.h', 'linux/safe_readlink.cc', 'linux/safe_readlink.h', + 'linux/symbol_collector_client.cc', + 'linux/symbol_collector_client.h', 'linux/synth_elf.cc', 'linux/synth_elf.h', 'long_string_dictionary.cc', diff --git a/src/common/linux/libcurl_wrapper.cc b/src/common/linux/libcurl_wrapper.cc index fd4e34cd..d935174b 100644 --- a/src/common/linux/libcurl_wrapper.cc +++ b/src/common/linux/libcurl_wrapper.cc @@ -38,32 +38,24 @@ namespace google_breakpad { LibcurlWrapper::LibcurlWrapper() : init_ok_(false), - formpost_(NULL), - lastptr_(NULL), - headerlist_(NULL) { - curl_lib_ = dlopen("libcurl.so", RTLD_NOW); - if (!curl_lib_) { - curl_lib_ = dlopen("libcurl.so.4", RTLD_NOW); - } - if (!curl_lib_) { - curl_lib_ = dlopen("libcurl.so.3", RTLD_NOW); - } - if (!curl_lib_) { - std::cout << "Could not find libcurl via dlopen"; - return; - } - std::cout << "LibcurlWrapper init succeeded"; - init_ok_ = true; - return; -} + curl_lib_(nullptr), + last_curl_error_(""), + curl_(nullptr), + formpost_(nullptr), + lastptr_(nullptr), + headerlist_(nullptr) {} -LibcurlWrapper::~LibcurlWrapper() {} +LibcurlWrapper::~LibcurlWrapper() { + if (init_ok_) { + (*easy_cleanup_)(curl_); + dlclose(curl_lib_); + } +} bool LibcurlWrapper::SetProxy(const string& proxy_host, const string& proxy_userpwd) { - if (!init_ok_) { - return false; - } + if (!CheckInit()) return false; + // Set proxy information if necessary. if (!proxy_host.empty()) { (*easy_setopt_)(curl_, CURLOPT_PROXY, proxy_host.c_str()); @@ -83,9 +75,8 @@ bool LibcurlWrapper::SetProxy(const string& proxy_host, bool LibcurlWrapper::AddFile(const string& upload_file_path, const string& basename) { - if (!init_ok_) { - return false; - } + if (!CheckInit()) return false; + std::cout << "Adding " << upload_file_path << " to form upload."; // Add form file. (*formadd_)(&formpost_, &lastptr_, @@ -110,10 +101,11 @@ static size_t WriteCallback(void *ptr, size_t size, bool LibcurlWrapper::SendRequest(const string& url, const std::map& parameters, - int* http_status_code, + long* http_status_code, string* http_header_data, string* http_response_data) { - (*easy_setopt_)(curl_, CURLOPT_URL, url.c_str()); + if (!CheckInit()) return false; + std::map::const_iterator iter = parameters.begin(); for (; iter != parameters.end(); ++iter) (*formadd_)(&formpost_, &lastptr_, @@ -122,55 +114,79 @@ bool LibcurlWrapper::SendRequest(const string& url, CURLFORM_END); (*easy_setopt_)(curl_, CURLOPT_HTTPPOST, formpost_); - if (http_response_data != NULL) { - http_response_data->clear(); - (*easy_setopt_)(curl_, CURLOPT_WRITEFUNCTION, WriteCallback); - (*easy_setopt_)(curl_, CURLOPT_WRITEDATA, - reinterpret_cast(http_response_data)); - } - if (http_header_data != NULL) { - http_header_data->clear(); - (*easy_setopt_)(curl_, CURLOPT_HEADERFUNCTION, WriteCallback); - (*easy_setopt_)(curl_, CURLOPT_HEADERDATA, - reinterpret_cast(http_header_data)); + + return SendRequestInner(url, http_status_code, http_header_data, + http_response_data); +} + +bool LibcurlWrapper::SendGetRequest(const string& url, + long* http_status_code, + string* http_header_data, + string* http_response_data) { + if (!CheckInit()) return false; + + (*easy_setopt_)(curl_, CURLOPT_HTTPGET, 1L); + + return SendRequestInner(url, http_status_code, http_header_data, + http_response_data); +} + +bool LibcurlWrapper::SendPutRequest(const string& url, + const string& path, + long* http_status_code, + string* http_header_data, + string* http_response_data) { + if (!CheckInit()) return false; + + FILE* file = fopen(path.c_str(), "rb"); + (*easy_setopt_)(curl_, CURLOPT_UPLOAD, 1L); + (*easy_setopt_)(curl_, CURLOPT_PUT, 1L); + (*easy_setopt_)(curl_, CURLOPT_READDATA, file); + + bool success = SendRequestInner(url, http_status_code, http_header_data, + http_response_data); + + fclose(file); + return success; +} + +bool LibcurlWrapper::SendSimplePostRequest(const string& url, + const string& body, + const string& content_type, + long* http_status_code, + string* http_header_data, + string* http_response_data) { + if (!CheckInit()) return false; + + (*easy_setopt_)(curl_, CURLOPT_POSTFIELDSIZE, body.size()); + (*easy_setopt_)(curl_, CURLOPT_COPYPOSTFIELDS, body.c_str()); + + if (!content_type.empty()) { + string content_type_header = "Content-Type: " + content_type; + headerlist_ = (*slist_append_)( + headerlist_, + content_type_header.c_str()); } - CURLcode err_code = CURLE_OK; - err_code = (*easy_perform_)(curl_); - easy_strerror_ = reinterpret_cast - (dlsym(curl_lib_, "curl_easy_strerror")); - - if (http_status_code != NULL) { - (*easy_getinfo_)(curl_, CURLINFO_RESPONSE_CODE, http_status_code); - } - -#ifndef NDEBUG - if (err_code != CURLE_OK) - fprintf(stderr, "Failed to send http request to %s, error: %s\n", - url.c_str(), - (*easy_strerror_)(err_code)); -#endif - if (headerlist_ != NULL) { - (*slist_free_all_)(headerlist_); - } - - (*easy_cleanup_)(curl_); - if (formpost_ != NULL) { - (*formfree_)(formpost_); - } - - return err_code == CURLE_OK; + return SendRequestInner(url, http_status_code, http_header_data, + http_response_data); } bool LibcurlWrapper::Init() { - if (!init_ok_) { - std::cout << "Init_OK was not true in LibcurlWrapper::Init(), check earlier log messages"; + curl_lib_ = dlopen("libcurl.so", RTLD_NOW); + if (!curl_lib_) { + curl_lib_ = dlopen("libcurl.so.4", RTLD_NOW); + } + if (!curl_lib_) { + curl_lib_ = dlopen("libcurl.so.3", RTLD_NOW); + } + if (!curl_lib_) { + std::cout << "Could not find libcurl via dlopen"; return false; } if (!SetFunctionPointers()) { std::cout << "Could not find function pointers"; - init_ok_ = false; return false; } @@ -184,11 +200,7 @@ bool LibcurlWrapper::Init() { return false; } - // Disable 100-continue header. - char buf[] = "Expect:"; - - headerlist_ = (*slist_append_)(headerlist_, buf); - (*easy_setopt_)(curl_, CURLOPT_HTTPHEADER, headerlist_); + init_ok_ = true; return true; } @@ -228,6 +240,10 @@ bool LibcurlWrapper::SetFunctionPointers() { "curl_easy_getinfo", CURLcode(*)(CURL *, CURLINFO info, ...)); + SET_AND_CHECK_FUNCTION_POINTER(easy_reset_, + "curl_easy_reset", + void(*)(CURL*)); + SET_AND_CHECK_FUNCTION_POINTER(slist_free_all_, "curl_slist_free_all", void(*)(curl_slist*)); @@ -238,4 +254,73 @@ bool LibcurlWrapper::SetFunctionPointers() { return true; } +bool LibcurlWrapper::SendRequestInner(const string& url, + long* http_status_code, + string* http_header_data, + string* http_response_data) { + string url_copy(url); + (*easy_setopt_)(curl_, CURLOPT_URL, url_copy.c_str()); + + // Disable 100-continue header. + char buf[] = "Expect:"; + headerlist_ = (*slist_append_)(headerlist_, buf); + (*easy_setopt_)(curl_, CURLOPT_HTTPHEADER, headerlist_); + + if (http_response_data != nullptr) { + http_response_data->clear(); + (*easy_setopt_)(curl_, CURLOPT_WRITEFUNCTION, WriteCallback); + (*easy_setopt_)(curl_, CURLOPT_WRITEDATA, + reinterpret_cast(http_response_data)); + } + if (http_header_data != nullptr) { + http_header_data->clear(); + (*easy_setopt_)(curl_, CURLOPT_HEADERFUNCTION, WriteCallback); + (*easy_setopt_)(curl_, CURLOPT_HEADERDATA, + reinterpret_cast(http_header_data)); + } + CURLcode err_code = CURLE_OK; + err_code = (*easy_perform_)(curl_); + easy_strerror_ = reinterpret_cast + (dlsym(curl_lib_, "curl_easy_strerror")); + + if (http_status_code != nullptr) { + (*easy_getinfo_)(curl_, CURLINFO_RESPONSE_CODE, http_status_code); + } + +#ifndef NDEBUG + if (err_code != CURLE_OK) + fprintf(stderr, "Failed to send http request to %s, error: %s\n", + url.c_str(), + (*easy_strerror_)(err_code)); +#endif + + Reset(); + + return err_code == CURLE_OK; } + +void LibcurlWrapper::Reset() { + if (headerlist_ != nullptr) { + (*slist_free_all_)(headerlist_); + headerlist_ = nullptr; + } + + if (formpost_ != nullptr) { + (*formfree_)(formpost_); + formpost_ = nullptr; + } + + (*easy_reset_)(curl_); +} + +bool LibcurlWrapper::CheckInit() { + if (!init_ok_) { + std::cout << "LibcurlWrapper: You must call Init(), and have it return " + "'true' before invoking any other methods.\n"; + return false; + } + + return true; +} + +} // namespace google_breakpad diff --git a/src/common/linux/libcurl_wrapper.h b/src/common/linux/libcurl_wrapper.h index 25905ad8..77aa6cbb 100644 --- a/src/common/linux/libcurl_wrapper.h +++ b/src/common/linux/libcurl_wrapper.h @@ -51,14 +51,39 @@ class LibcurlWrapper { const string& basename); virtual bool SendRequest(const string& url, const std::map& parameters, - int* http_status_code, + long* http_status_code, string* http_header_data, string* http_response_data); + bool SendGetRequest(const string& url, + long* http_status_code, + string* http_header_data, + string* http_response_data); + bool SendPutRequest(const string& url, + const string& path, + long* http_status_code, + string* http_header_data, + string* http_response_data); + bool SendSimplePostRequest(const string& url, + const string& body, + const string& content_type, + long* http_status_code, + string* http_header_data, + string* http_response_data); + private: // This function initializes class state corresponding to function // pointers into the CURL library. bool SetFunctionPointers(); + bool SendRequestInner(const string& url, + long* http_status_code, + string* http_header_data, + string* http_response_data); + + void Reset(); + + bool CheckInit(); + bool init_ok_; // Whether init succeeded void* curl_lib_; // Pointer to result of dlopen() on // curl library @@ -85,6 +110,7 @@ class LibcurlWrapper { const char* (*easy_strerror_)(CURLcode); void (*easy_cleanup_)(CURL *); CURLcode (*easy_getinfo_)(CURL *, CURLINFO info, ...); + void (*easy_reset_)(CURL*); void (*formfree_)(struct curl_httppost *); }; diff --git a/src/common/linux/symbol_collector_client.cc b/src/common/linux/symbol_collector_client.cc new file mode 100644 index 00000000..ea995c4b --- /dev/null +++ b/src/common/linux/symbol_collector_client.cc @@ -0,0 +1,193 @@ +// Copyright (c) 2019 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "common/linux/symbol_collector_client.h" + +#include + +#include +#include + +#include "common/linux/libcurl_wrapper.h" + +namespace google_breakpad { +namespace sym_upload { + +// static +bool SymbolCollectorClient::CreateUploadUrl( + LibcurlWrapper* libcurl_wrapper, + const string& api_url, + const string& api_key, + UploadUrlResponse* uploadUrlResponse) { + string header, response; + long response_code; + + string url = api_url + "/v1/uploads:create"; + if (!api_key.empty()) { + url += "?key=" + api_key; + } + + if (!libcurl_wrapper->SendSimplePostRequest(url, + /*body=*/"", + /*content_type=*/"", + &response_code, + &header, + &response)) { + printf("Failed to create upload url.\n"); + printf("Response code: %ld\n", response_code); + printf("Response:\n"); + printf("%s\n", response.c_str()); + return false; + } + + // Note camel-case rather than underscores. + std::regex upload_url_regex("\"uploadUrl\": \"([^\"]+)\""); + std::regex upload_key_regex("\"uploadKey\": \"([^\"]+)\""); + + std::smatch upload_url_match; + if (!std::regex_search(response, upload_url_match, upload_url_regex) || + upload_url_match.size() != 2) { + printf("Failed to parse create url response."); + printf("Response:\n"); + printf("%s\n", response.c_str()); + return false; + } + string upload_url = upload_url_match[1].str(); + + std::smatch upload_key_match; + if (!std::regex_search(response, upload_key_match, upload_key_regex) || + upload_key_match.size() != 2) { + printf("Failed to parse create url response."); + printf("Response:\n"); + printf("%s\n", response.c_str()); + return false; + } + string upload_key = upload_key_match[1].str(); + + uploadUrlResponse->upload_url = upload_url; + uploadUrlResponse->upload_key = upload_key; + return true; +} + +// static +CompleteUploadResult SymbolCollectorClient::CompleteUpload( + LibcurlWrapper* libcurl_wrapper, + const string& api_url, + const string& api_key, + const string& upload_key, + const string& debug_file, + const string& debug_id) { + string header, response; + long response_code; + + string url = api_url + "/v1/uploads/" + upload_key + ":complete"; + if (!api_key.empty()) { + url += "?key=" + api_key; + } + string body = + "{ symbol_id: {" + "debug_file: \"" + debug_file + "\", " + "debug_id: \"" + debug_id + "\" } }"; + + if (!libcurl_wrapper->SendSimplePostRequest(url, + body, + "application/son", + &response_code, + &header, + &response)) { + printf("Failed to complete upload.\n"); + printf("Response code: %ld\n", response_code); + printf("Response:\n"); + printf("%s\n", response.c_str()); + return CompleteUploadResult::Error; + } + + std::regex result_regex("\"result\": \"([^\"]+)\""); + std::smatch result_match; + if (!std::regex_search(response, result_match, result_regex) || + result_match.size() != 2) { + printf("Failed to parse complete upload response."); + printf("Response:\n"); + printf("%s\n", response.c_str()); + return CompleteUploadResult::Error; + } + string result = result_match[1].str(); + + if (result.compare("DUPLICATE_DATA") == 0) { + return CompleteUploadResult::DuplicateData; + } + + return CompleteUploadResult::Ok; +} + +// static +SymbolStatus SymbolCollectorClient::CheckSymbolStatus( + LibcurlWrapper* libcurl_wrapper, + const string& api_url, + const string& api_key, + const string& debug_file, + const string& debug_id) { + string header, response; + long response_code; + string url = api_url + + "/v1/symbols/" + debug_file + "/" + debug_id + ":checkStatus"; + if (!api_key.empty()) { + url += "?key=" + api_key; + } + + if (!libcurl_wrapper->SendGetRequest( + url, + &response_code, + &header, + &response)) { + printf("Failed to check symbol status, error message.\n"); + printf("Response code: %ld\n", response_code); + printf("Response:\n"); + printf("%s\n", response.c_str()); + return SymbolStatus::Unknown; + } + + std::regex status_regex("\"status\": \"([^\"]+)\""); + std::smatch status_match; + if (!std::regex_search(response, status_match, status_regex) || + status_match.size() != 2) { + printf("Failed to parse check symbol status response."); + printf("Response:\n"); + printf("%s\n", response.c_str()); + return SymbolStatus::Unknown; + } + string status = status_match[1].str(); + + return (status.compare("FOUND") == 0) ? + SymbolStatus::Found : + SymbolStatus::Missing; +} + +} // namespace sym_upload +} // namespace google_breakpad diff --git a/src/common/linux/symbol_collector_client.h b/src/common/linux/symbol_collector_client.h new file mode 100644 index 00000000..5f811de4 --- /dev/null +++ b/src/common/linux/symbol_collector_client.h @@ -0,0 +1,87 @@ +// Copyright (c) 2019, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_ +#define COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_ + +#include + +#include "common/linux/libcurl_wrapper.h" +#include "common/using_std_string.h" + +namespace google_breakpad { +namespace sym_upload { + +struct UploadUrlResponse { + string upload_url; + string upload_key; +}; + +enum SymbolStatus { + Found, + Missing, + Unknown +}; + +enum CompleteUploadResult { + Ok, + DuplicateData, + Error +}; + +// Helper class to communicate with a sym-upload-v2 service over HTTP/REST, +// via libcurl. +class SymbolCollectorClient { + public: + static bool CreateUploadUrl( + LibcurlWrapper* libcurl_wrapper, + const string& api_url, + const string& api_key, + UploadUrlResponse* uploadUrlResponse); + + static CompleteUploadResult CompleteUpload( + LibcurlWrapper* libcurl_wrapper, + const string& api_url, + const string& api_key, + const string& upload_key, + const string& debug_file, + const string& debug_id); + + static SymbolStatus CheckSymbolStatus( + LibcurlWrapper* libcurl_wrapper, + const string& api_url, + const string& api_key, + const string& debug_file, + const string& debug_id); +}; + +} // namespace sym_upload +} // namespace google_breakpad + +#endif // COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_ diff --git a/src/common/linux/symbol_upload.cc b/src/common/linux/symbol_upload.cc index bbd3181e..99750fd1 100644 --- a/src/common/linux/symbol_upload.cc +++ b/src/common/linux/symbol_upload.cc @@ -30,15 +30,19 @@ // symbol_upload.cc: implemented google_breakpad::sym_upload::Start, a helper // function for linux symbol upload tool. -#include "common/linux/http_upload.h" #include "common/linux/symbol_upload.h" #include #include #include +#include #include +#include "common/linux/http_upload.h" +#include "common/linux/libcurl_wrapper.h" +#include "common/linux/symbol_collector_client.h" + namespace google_breakpad { namespace sym_upload { @@ -95,21 +99,19 @@ string CompactIdentifier(const string &uuid) { return result; } -//============================================================================= -void Start(Options *options) { +// |options| describes the current sym_upload options. +// |module_parts| contains the strings parsed from the MODULE entry of the +// Breakpad symbol file being uploaded. +// |compacted_id| is the debug_id from the MODULE entry of the Breakpad symbol +// file being uploaded, with all hyphens removed. +bool SymUploadV1Start( + const Options& options, + std::vector module_parts, + const string& compacted_id) { std::map parameters; - options->success = false; - std::vector module_parts; - if (!ModuleDataForSymbolFile(options->symbolsPath, &module_parts)) { - fprintf(stderr, "Failed to parse symbol file!\n"); - return; - } - - string compacted_id = CompactIdentifier(module_parts[3]); - // Add parameters - if (!options->version.empty()) - parameters["version"] = options->version; + if (!options.version.empty()) + parameters["version"] = options.version; // MODULE // 0 1 2 3 4 @@ -120,16 +122,16 @@ void Start(Options *options) { parameters["debug_identifier"] = compacted_id; std::map files; - files["symbol_file"] = options->symbolsPath; + files["symbol_file"] = options.symbolsPath; string response, error; long response_code; - bool success = HTTPUpload::SendRequest(options->uploadURLStr, + bool success = HTTPUpload::SendRequest(options.uploadURLStr, parameters, files, - options->proxy, - options->proxy_user_pwd, - "", + options.proxy, + options.proxy_user_pwd, + /*ca_certificate_file=*/"", &response, &response_code, &error); @@ -148,7 +150,116 @@ void Start(Options *options) { } else { printf("Successfully sent the symbol file.\n"); } - options->success = success; + + return success; +} + +// |options| describes the current sym_upload options. +// |module_parts| contains the strings parsed from the MODULE entry of the +// Breakpad symbol file being uploaded. +// |compacted_id| is the debug_id from the MODULE entry of the Breakpad symbol +// file being uploaded, with all hyphens removed. +bool SymUploadV2Start( + const Options& options, + std::vector module_parts, + const string& compacted_id) { + string debug_file = module_parts[4]; + string debug_id = compacted_id; + + google_breakpad::LibcurlWrapper libcurl_wrapper; + if (!libcurl_wrapper.Init()) { + printf("Failed to init google_breakpad::LibcurlWrapper.\n"); + return false; + } + + if (!options.force) { + SymbolStatus symbolStatus = SymbolCollectorClient::CheckSymbolStatus( + &libcurl_wrapper, + options.uploadURLStr, + options.api_key, + debug_file, + debug_id); + if (symbolStatus == SymbolStatus::Found) { + printf("Symbol file already exists, upload aborted." + " Use \"-f\" to overwrite.\n"); + return true; + } else if (symbolStatus == SymbolStatus::Unknown) { + printf("Failed to check for existing symbol.\n"); + return false; + } + } + + UploadUrlResponse uploadUrlResponse; + if (!SymbolCollectorClient::CreateUploadUrl( + &libcurl_wrapper, + options.uploadURLStr, + options.api_key, + &uploadUrlResponse)) { + printf("Failed to create upload URL.\n"); + return false; + } + + string signed_url = uploadUrlResponse.upload_url; + string upload_key = uploadUrlResponse.upload_key; + string header; + string response; + long response_code; + + if (!libcurl_wrapper.SendPutRequest(signed_url, + options.symbolsPath, + &response_code, + &header, + &response)) { + printf("Failed to send symbol file.\n"); + printf("Response code: %ld\n", response_code); + printf("Response:\n"); + printf("%s\n", response.c_str()); + return false; + } else if (response_code == 0) { + printf("Failed to send symbol file: No response code\n"); + return false; + } else if (response_code != 200) { + printf("Failed to send symbol file: Response code %ld\n", response_code); + printf("Response:\n"); + printf("%s\n", response.c_str()); + return false; + } + + CompleteUploadResult completeUploadResult = + SymbolCollectorClient::CompleteUpload(&libcurl_wrapper, + options.uploadURLStr, + options.api_key, + upload_key, + debug_file, + debug_id); + if (completeUploadResult == CompleteUploadResult::Error) { + printf("Failed to complete upload.\n"); + return false; + } else if (completeUploadResult == CompleteUploadResult::DuplicateData) { + printf("Uploaded file checksum matched existing file checksum," + " no change necessary.\n"); + } else { + printf("Successfully sent the symbol file.\n"); + } + + return true; +} + +//============================================================================= +void Start(Options* options) { + std::vector module_parts; + if (!ModuleDataForSymbolFile(options->symbolsPath, &module_parts)) { + fprintf(stderr, "Failed to parse symbol file!\n"); + return; + } + + const string compacted_id = CompactIdentifier(module_parts[3]); + + if (options->upload_protocol == UploadProtocol::SYM_UPLOAD_V2) { + options->success = SymUploadV2Start(*options, module_parts, compacted_id); + } else { + options->success = SymUploadV1Start(*options, module_parts, compacted_id); + } } } // namespace sym_upload diff --git a/src/common/linux/symbol_upload.h b/src/common/linux/symbol_upload.h index 0a469692..040e980f 100644 --- a/src/common/linux/symbol_upload.h +++ b/src/common/linux/symbol_upload.h @@ -41,14 +41,24 @@ namespace google_breakpad { namespace sym_upload { -typedef struct { +enum class UploadProtocol { + SYM_UPLOAD_V1, + SYM_UPLOAD_V2, +}; + +struct Options { + Options() : upload_protocol(UploadProtocol::SYM_UPLOAD_V1), force(false) {} + string symbolsPath; string uploadURLStr; string proxy; string proxy_user_pwd; string version; bool success; -} Options; + UploadProtocol upload_protocol; + bool force; + string api_key; +}; // Starts upload to symbol server with options. void Start(Options* options); diff --git a/src/tools/linux/symupload/sym_upload.cc b/src/tools/linux/symupload/sym_upload.cc index 9eeb2d44..cb5321a7 100644 --- a/src/tools/linux/symupload/sym_upload.cc +++ b/src/tools/linux/symupload/sym_upload.cc @@ -41,25 +41,43 @@ #include #include +#include #include #include "common/linux/symbol_upload.h" +using google_breakpad::sym_upload::UploadProtocol; using google_breakpad::sym_upload::Options; //============================================================================= static void Usage(int argc, const char *argv[]) { fprintf(stderr, "Submit symbol information.\n"); - fprintf(stderr, "Usage: %s [options...] \n", argv[0]); + fprintf(stderr, "Usage: %s [options...] \n", + argv[0]); fprintf(stderr, "Options:\n"); - fprintf(stderr, " should be created by using the dump_syms tool.\n"); + fprintf(stderr, " should be created by using the dump_syms" + "tool.\n"); fprintf(stderr, " is the destination for the upload\n"); + fprintf(stderr, "-p:\t One of ['sym-upload-v1'," + " 'sym-upload-v2'], defaults to 'sym-upload-v1'.\n"); + fprintf(stderr, "-k:\t A secret used to authenticate with the" + " API [Only supported when using 'sym-upload-v2' protocol].\n"); + fprintf(stderr, "-f:\t Force symbol upload if already exists [Only" + " supported when using 'sym-upload-v2' protocol].\n"); fprintf(stderr, "-v:\t Version information (e.g., 1.2.3.4)\n"); fprintf(stderr, "-x:\t Use HTTP proxy on given port\n"); fprintf(stderr, "-u:\t Set proxy user and password\n"); fprintf(stderr, "-h:\t Usage\n"); fprintf(stderr, "-?:\t Usage\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "Examples:\n"); + fprintf(stderr, " With 'sym-upload-v1':\n"); + fprintf(stderr, " %s path/to/symbol_file http://myuploadserver\n", + argv[0]); + fprintf(stderr, " With 'sym-upload-v2':\n"); + fprintf(stderr, " %s -p sym-upload-v2 -k mysecret123! " + "path/to/symbol_file http://myuploadserver\n", argv[0]); } //============================================================================= @@ -68,7 +86,7 @@ SetupOptions(int argc, const char *argv[], Options *options) { extern int optind; int ch; - while ((ch = getopt(argc, (char * const *)argv, "u:v:x:h?")) != -1) { + while ((ch = getopt(argc, (char * const *)argv, "u:v:x:p:k:hf?")) != -1) { switch (ch) { case 'h': case '?': @@ -84,6 +102,23 @@ SetupOptions(int argc, const char *argv[], Options *options) { case 'x': options->proxy = optarg; break; + case 'p': + if (strcmp(optarg, "sym-upload-v2") == 0) { + options->upload_protocol = UploadProtocol::SYM_UPLOAD_V2; + } else if (strcmp(optarg, "sym-upload-v1") == 0) { + options->upload_protocol = UploadProtocol::SYM_UPLOAD_V1; + } else { + fprintf(stderr, "Invalid protocol '%c'\n", optarg); + Usage(argc, argv); + exit(1); + } + break; + case 'k': + options->api_key = optarg; + break; + case 'f': + options->force = true; + break; default: fprintf(stderr, "Invalid option '%c'\n", ch);