From 20eff509e89013ba3a4705f4384a07e59fe4513f Mon Sep 17 00:00:00 2001 From: Patrick Lodder Date: Thu, 12 Jan 2023 17:57:55 +0100 Subject: [PATCH 1/7] devtools: backport lief-based security and symbol checkers Takes the security and symbol checkers from Bitcoin Core v24.0.1 because this uses the python3 capable lief module for reading multi-platform binaries. This helps getting rid of incompatibilities when using these tools in Ubuntu releases newer than Bionic (18.04) and by using the external module, reduces risk and maintenance cost of custom code. This commit does NOT reconfigure for Dogecoin 1.14.7 parametrization Backported from state at: b3f866a8@bitcoin/bitcoin --- contrib/devtools/security-check.py | 379 +++++++++++++----------- contrib/devtools/symbol-check.py | 374 +++++++++++++++-------- contrib/devtools/test-security-check.py | 139 +++++++-- contrib/devtools/test-symbol-check.py | 204 +++++++++++++ contrib/devtools/utils.py | 22 ++ 5 files changed, 799 insertions(+), 319 deletions(-) create mode 100755 contrib/devtools/test-symbol-check.py create mode 100755 contrib/devtools/utils.py diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index c90541e27..05c0af029 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,216 +1,263 @@ -#!/usr/bin/env python -# Copyright (c) 2015-2016 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' -Perform basic ELF security checks on a series of executables. +Perform basic security checks on a series of executables. Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. -Needs `readelf` (for ELF) and `objdump` (for PE). ''' -from __future__ import division,print_function,unicode_literals -import subprocess import sys -import os +from typing import List -READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') -OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') -NONFATAL = {'HIGH_ENTROPY_VA'} # checks which are non-fatal for now but only generate a warning +import lief #type:ignore -def check_ELF_PIE(executable): - ''' - Check for position independent executable (PIE), allowing for address space randomization. - ''' - p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - - ok = False - for line in stdout.split(b'\n'): - line = line.split() - if len(line)>=2 and line[0] == b'Type:' and line[1] == b'DYN': - ok = True - return ok - -def get_ELF_program_headers(executable): - '''Return type and flags for ELF program headers''' - p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - in_headers = False - count = 0 - headers = [] - for line in stdout.split(b'\n'): - if line.startswith(b'Program Headers:'): - in_headers = True - if line == b'': - in_headers = False - if in_headers: - if count == 1: # header line - ofs_typ = line.find(b'Type') - ofs_offset = line.find(b'Offset') - ofs_flags = line.find(b'Flg') - ofs_align = line.find(b'Align') - if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align == -1: - raise ValueError('Cannot parse elfread -lW output') - elif count > 1: - typ = line[ofs_typ:ofs_offset].rstrip() - flags = line[ofs_flags:ofs_align].rstrip() - headers.append((typ, flags)) - count += 1 - return headers - -def check_ELF_NX(executable): - ''' - Check that no sections are writable and executable (including the stack) - ''' - have_wx = False - have_gnu_stack = False - for (typ, flags) in get_ELF_program_headers(executable): - if typ == b'GNU_STACK': - have_gnu_stack = True - if b'W' in flags and b'E' in flags: # section is both writable and executable - have_wx = True - return have_gnu_stack and not have_wx - -def check_ELF_RELRO(executable): +def check_ELF_RELRO(binary) -> bool: ''' Check for read-only relocations. GNU_RELRO program header must exist Dynamic section must have BIND_NOW flag ''' have_gnu_relro = False - for (typ, flags) in get_ELF_program_headers(executable): - # Note: not checking flags == 'R': here as linkers set the permission differently - # This does not affect security: the permission flags of the GNU_RELRO program header are ignored, the PT_LOAD header determines the effective permissions. + for segment in binary.segments: + # Note: not checking p_flags == PF_R: here as linkers set the permission differently + # This does not affect security: the permission flags of the GNU_RELRO program + # header are ignored, the PT_LOAD header determines the effective permissions. # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. - # See also http://permalink.gmane.org/gmane.comp.gnu.binutils/71347 - if typ == b'GNU_RELRO': + # See also https://marc.info/?l=binutils&m=1498883354122353 + if segment.type == lief.ELF.SEGMENT_TYPES.GNU_RELRO: have_gnu_relro = True have_bindnow = False - p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - for line in stdout.split(b'\n'): - tokens = line.split() - if len(tokens)>1 and tokens[1] == b'(BIND_NOW)' or (len(tokens)>2 and tokens[1] == b'(FLAGS)' and b'BIND_NOW' in tokens[2]): + try: + flags = binary.get(lief.ELF.DYNAMIC_TAGS.FLAGS) + if flags.value & lief.ELF.DYNAMIC_FLAGS.BIND_NOW: have_bindnow = True + except: + have_bindnow = False + return have_gnu_relro and have_bindnow -def check_ELF_Canary(executable): +def check_ELF_Canary(binary) -> bool: ''' Check for use of stack canary ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - ok = False - for line in stdout.split(b'\n'): - if b'__stack_chk_fail' in line: - ok = True - return ok + return binary.has_symbol('__stack_chk_fail') -def get_PE_dll_characteristics(executable): +def check_ELF_separate_code(binary): ''' - Get PE DllCharacteristics bits. - Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386' - and bits is the DllCharacteristics value. + Check that sections are appropriately separated in virtual memory, + based on their permissions. This checks for missing -Wl,-z,separate-code + and potentially other problems. ''' - p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - arch = '' - bits = 0 - for line in stdout.split('\n'): - tokens = line.split() - if len(tokens)>=2 and tokens[0] == 'architecture:': - arch = tokens[1].rstrip(',') - if len(tokens)>=2 and tokens[0] == 'DllCharacteristics': - bits = int(tokens[1],16) - return (arch,bits) + R = lief.ELF.SEGMENT_FLAGS.R + W = lief.ELF.SEGMENT_FLAGS.W + E = lief.ELF.SEGMENT_FLAGS.X + EXPECTED_FLAGS = { + # Read + execute + '.init': R | E, + '.plt': R | E, + '.plt.got': R | E, + '.plt.sec': R | E, + '.text': R | E, + '.fini': R | E, + # Read-only data + '.interp': R, + '.note.gnu.property': R, + '.note.gnu.build-id': R, + '.note.ABI-tag': R, + '.gnu.hash': R, + '.dynsym': R, + '.dynstr': R, + '.gnu.version': R, + '.gnu.version_r': R, + '.rela.dyn': R, + '.rela.plt': R, + '.rodata': R, + '.eh_frame_hdr': R, + '.eh_frame': R, + '.qtmetadata': R, + '.gcc_except_table': R, + '.stapsdt.base': R, + # Writable data + '.init_array': R | W, + '.fini_array': R | W, + '.dynamic': R | W, + '.got': R | W, + '.data': R | W, + '.bss': R | W, + } + if binary.header.machine_type == lief.ELF.ARCH.PPC64: + # .plt is RW on ppc64 even with separate-code + EXPECTED_FLAGS['.plt'] = R | W + # For all LOAD program headers get mapping to the list of sections, + # and for each section, remember the flags of the associated program header. + flags_per_section = {} + for segment in binary.segments: + if segment.type == lief.ELF.SEGMENT_TYPES.LOAD: + for section in segment.sections: + flags_per_section[section.name] = segment.flags + # Spot-check ELF LOAD program header flags per section + # If these sections exist, check them against the expected R/W/E flags + for (section, flags) in flags_per_section.items(): + if section in EXPECTED_FLAGS: + if int(EXPECTED_FLAGS[section]) != int(flags): + return False + return True -IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020 -IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE = 0x0040 -IMAGE_DLL_CHARACTERISTICS_NX_COMPAT = 0x0100 +def check_ELF_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_function_address('main') + content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) -def check_PE_DYNAMIC_BASE(executable): + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_PE_DYNAMIC_BASE(binary) -> bool: '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - (arch,bits) = get_PE_dll_characteristics(executable) - reqbits = IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE - return (bits & reqbits) == reqbits + return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists -# On 64 bit, must support high-entropy 64-bit address space layout randomization in addition to DYNAMIC_BASE -# to have secure ASLR. -def check_PE_HIGH_ENTROPY_VA(executable): +# Must support high-entropy 64-bit address space layout randomization +# in addition to DYNAMIC_BASE to have secure ASLR. +def check_PE_HIGH_ENTROPY_VA(binary) -> bool: '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - (arch,bits) = get_PE_dll_characteristics(executable) - if arch == 'i386:x86-64': - reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA - else: # Unnecessary on 32-bit - assert(arch == 'i386') - reqbits = 0 - return (bits & reqbits) == reqbits + return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists -def check_PE_NX(executable): - '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' - (arch,bits) = get_PE_dll_characteristics(executable) - return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT +def check_PE_RELOC_SECTION(binary) -> bool: + '''Check for a reloc section. This is required for functional ASLR.''' + return binary.has_relocations -CHECKS = { -'ELF': [ - ('PIE', check_ELF_PIE), - ('NX', check_ELF_NX), +def check_PE_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_symbol('main').value + + section_addr = binary.section_from_rva(main).virtual_address + virtual_address = binary.optional_header.imagebase + section_addr + main + + content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_MACHO_NOUNDEFS(binary) -> bool: + ''' + Check for no undefined references. + ''' + return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) + +def check_MACHO_LAZY_BINDINGS(binary) -> bool: + ''' + Check for no lazy bindings. + We don't use or check for MH_BINDATLOAD. See #18295. + ''' + return binary.dyld_info.lazy_bind == (0,0) + +def check_MACHO_Canary(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('___stack_chk_fail') + +def check_PIE(binary) -> bool: + ''' + Check for position independent executable (PIE), + allowing for address space randomization. + ''' + return binary.is_pie + +def check_NX(binary) -> bool: + ''' + Check for no stack execution + ''' + return binary.has_nx + +def check_MACHO_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + +BASE_ELF = [ + ('PIE', check_PIE), + ('NX', check_NX), ('RELRO', check_ELF_RELRO), - ('Canary', check_ELF_Canary) -], -'PE': [ + ('Canary', check_ELF_Canary), + ('separate_code', check_ELF_separate_code), +] + +BASE_PE = [ + ('PIE', check_PIE), ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), - ('NX', check_PE_NX) + ('NX', check_NX), + ('RELOC_SECTION', check_PE_RELOC_SECTION), + ('CONTROL_FLOW', check_PE_control_flow), ] + +BASE_MACHO = [ + ('NOUNDEFS', check_MACHO_NOUNDEFS), + ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS), + ('Canary', check_MACHO_Canary), +] + +CHECKS = { + lief.EXE_FORMATS.ELF: { + lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_control_flow)], + lief.ARCHITECTURES.ARM: BASE_ELF, + lief.ARCHITECTURES.ARM64: BASE_ELF, + lief.ARCHITECTURES.PPC: BASE_ELF, + lief.ARCHITECTURES.RISCV: BASE_ELF, + }, + lief.EXE_FORMATS.PE: { + lief.ARCHITECTURES.X86: BASE_PE, + }, + lief.EXE_FORMATS.MACHO: { + lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE), + ('NX', check_NX), + ('CONTROL_FLOW', check_MACHO_control_flow)], + lief.ARCHITECTURES.ARM64: BASE_MACHO, + } } -def identify_executable(executable): - with open(filename, 'rb') as f: - magic = f.read(4) - if magic.startswith(b'MZ'): - return 'PE' - elif magic.startswith(b'\x7fELF'): - return 'ELF' - return None - if __name__ == '__main__': - retval = 0 + retval: int = 0 for filename in sys.argv[1:]: try: - etype = identify_executable(filename) - if etype is None: - print('%s: unknown format' % filename) + binary = lief.parse(filename) + etype = binary.format + arch = binary.abstract.header.architecture + binary.concrete + + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') retval = 1 continue - failed = [] - warning = [] - for (name, func) in CHECKS[etype]: - if not func(filename): - if name in NONFATAL: - warning.append(name) - else: - failed.append(name) - if failed: - print('%s: failed %s' % (filename, ' '.join(failed))) + if arch == lief.ARCHITECTURES.NONE: + print(f'{filename}: unknown architecture') retval = 1 - if warning: - print('%s: warning %s' % (filename, ' '.join(warning))) - except IOError: - print('%s: cannot open' % filename) - retval = 1 - exit(retval) + continue + + failed: List[str] = [] + for (name, func) in CHECKS[etype][arch]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') + retval = 1 + except IOError: + print(f'{filename}: cannot open') + retval = 1 + sys.exit(retval) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index cee7ff269..4b1cceb57 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -1,166 +1,284 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' -A script to check that the (Linux) executables produced by gitian only contain -allowed gcc, glibc and libstdc++ version symbols. This makes sure they are -still compatible with the minimum supported Linux distribution versions. +A script to check that release executables only contain certain symbols +and are only linked against allowed libraries. Example usage: - find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py + find ../path/to/binaries -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' -from __future__ import division, print_function, unicode_literals -import subprocess -import re import sys -import os +from typing import List, Dict -# Debian 6.0.9 (Squeeze) has: +import lief #type:ignore + +# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases # -# - g++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=g%2B%2B) -# - libc version 2.11.3 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libc6) -# - libstdc++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libstdc%2B%2B6) +# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6) # -# Ubuntu 10.04.4 (Lucid Lynx) has: +# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases # -# - g++ version 4.4.3 (http://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=lucid§ion=all) -# - libc version 2.11.1 (http://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=lucid§ion=all) -# - libstdc++ version 4.4.3 (http://packages.ubuntu.com/search?suite=lucid§ion=all&arch=any&keywords=libstdc%2B%2B&searchon=names) +# - g++ version 5.3.1 +# - libc version 2.23 # -# Taking the minimum of these as our target. +# CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product # -# According to GNU ABI document (http://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: -# GCC 4.4.0: GCC_4.4.0 -# GCC 4.4.2: GLIBCXX_3.4.13, CXXABI_1.3.3 -# (glibc) GLIBC_2_11 +# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) +# - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) # +# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. + MAX_VERSIONS = { -'GCC': (4,4,0), -'CXXABI': (1,3,3), -'GLIBCXX': (3,4,13), -'GLIBC': (2,11), -'V': (0,5,0) # xkb (qt only) +'GCC': (4,8,0), +'GLIBC': { + lief.ELF.ARCH.x86_64: (2,18), + lief.ELF.ARCH.ARM: (2,18), + lief.ELF.ARCH.AARCH64:(2,18), + lief.ELF.ARCH.PPC64: (2,18), + lief.ELF.ARCH.RISCV: (2,27), +}, +'LIBATOMIC': (1,0), +'V': (0,5,0), # xkb (bitcoin-qt only) } -# See here for a description of _IO_stdin_used: -# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { - b'_edata', b'_end', b'_init', b'__bss_start', b'_fini', b'_IO_stdin_used', - b'stdin', b'stdout', b'stderr' +'environ', '_environ', '__environ', '_fini', '_init', 'stdin', +'stdout', 'stderr', } -READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') -CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') + +# Expected linker-loader names can be found here: +# https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 +ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { + lief.ELF.ARCH.x86_64: { + lief.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", + }, + lief.ELF.ARCH.ARM: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", + }, + lief.ELF.ARCH.AARCH64: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-aarch64.so.1", + }, + lief.ELF.ARCH.PPC64: { + lief.ENDIANNESS.BIG: "/lib64/ld64.so.1", + lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", + }, + lief.ELF.ARCH.RISCV: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", + }, +} + # Allowed NEEDED libraries -ALLOWED_LIBRARIES = { +ELF_ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt -b'libgcc_s.so.1', # GCC base support -b'libc.so.6', # C library -b'libpthread.so.0', # threading -b'libanl.so.1', # DNS resolve -b'libm.so.6', # math library -b'librt.so.1', # real-time (clock) -b'ld-linux-x86-64.so.2', # 64-bit dynamic linker -b'ld-linux.so.2', # 32-bit dynamic linker +'libgcc_s.so.1', # GCC base support +'libc.so.6', # C library +'libpthread.so.0', # threading +'libm.so.6', # math library +'librt.so.1', # real-time (clock) +'libatomic.so.1', +'ld-linux-x86-64.so.2', # 64-bit dynamic linker +'ld-linux.so.2', # 32-bit dynamic linker +'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker +'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker +'ld64.so.1', # POWER64 ABIv1 dynamic linker +'ld64.so.2', # POWER64 ABIv2 dynamic linker +'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker # bitcoin-qt only -b'libX11-xcb.so.1', # part of X11 -b'libX11.so.6', # part of X11 -b'libxcb.so.1', # part of X11 -b'libxkbcommon.so.0', # keyboard keymapping -b'libxkbcommon-x11.so.0', # keyboard keymapping -b'libfontconfig.so.1', # font support -b'libfreetype.so.6', # font parsing -b'libdl.so.2' # programming interface to dynamic linker +'libxcb.so.1', # part of X11 +'libxkbcommon.so.0', # keyboard keymapping +'libxkbcommon-x11.so.0', # keyboard keymapping +'libfontconfig.so.1', # font support +'libfreetype.so.6', # font parsing +'libdl.so.2', # programming interface to dynamic linker +'libxcb-icccm.so.4', +'libxcb-image.so.0', +'libxcb-shm.so.0', +'libxcb-keysyms.so.1', +'libxcb-randr.so.0', +'libxcb-render-util.so.0', +'libxcb-render.so.0', +'libxcb-shape.so.0', +'libxcb-sync.so.1', +'libxcb-xfixes.so.0', +'libxcb-xinerama.so.0', +'libxcb-xkb.so.1', } -class CPPFilt(object): - ''' - Demangle C++ symbol names. +MACHO_ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libc++.1.dylib', # C++ Standard Library +'libSystem.B.dylib', # libc, libm, libpthread, libinfo +# bitcoin-qt only +'AppKit', # user interface +'ApplicationServices', # common application tasks. +'Carbon', # deprecated c back-compat API +'ColorSync', +'CoreFoundation', # low level func, data types +'CoreGraphics', # 2D rendering +'CoreServices', # operating system services +'CoreText', # interface for laying out text and handling fonts. +'CoreVideo', # video processing +'Foundation', # base layer functionality for apps/frameworks +'ImageIO', # read and write image file formats. +'IOKit', # user-space access to hardware devices and drivers. +'IOSurface', # cross process image/drawing buffers +'libobjc.A.dylib', # Objective-C runtime library +'Metal', # 3D graphics +'Security', # access control and authentication +'QuartzCore', # animation +} - Use a pipe to the 'c++filt' command. - ''' - def __init__(self): - self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE) +PE_ALLOWED_LIBRARIES = { +'ADVAPI32.dll', # security & registry +'IPHLPAPI.DLL', # IP helper API +'KERNEL32.dll', # win32 base APIs +'msvcrt.dll', # C standard library for MSVC +'SHELL32.dll', # shell API +'USER32.dll', # user interface +'WS2_32.dll', # sockets +# bitcoin-qt only +'dwmapi.dll', # desktop window manager +'GDI32.dll', # graphics device interface +'IMM32.dll', # input method editor +'NETAPI32.dll', +'ole32.dll', # component object model +'OLEAUT32.dll', # OLE Automation API +'SHLWAPI.dll', # light weight shell API +'USERENV.dll', +'UxTheme.dll', +'VERSION.dll', # version checking +'WINMM.dll', # WinMM audio API +'WTSAPI32.dll', +} - def __call__(self, mangled): - self.proc.stdin.write(mangled + b'\n') - self.proc.stdin.flush() - return self.proc.stdout.readline().rstrip() - - def close(self): - self.proc.stdin.close() - self.proc.stdout.close() - self.proc.wait() - -def read_symbols(executable, imports=True): - ''' - Parse an ELF executable and return a list of (symbol,version) tuples - for dynamic, imported symbols. - ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip())) - syms = [] - for line in stdout.split(b'\n'): - line = line.split() - if len(line)>7 and re.match(b'[0-9]+:$', line[0]): - (sym, _, version) = line[7].partition(b'@') - is_import = line[6] == b'UND' - if version.startswith(b'@'): - version = version[1:] - if is_import == imports: - syms.append((sym, version)) - return syms - -def check_version(max_versions, version): - if b'_' in version: - (lib, _, ver) = version.rpartition(b'_') - else: - lib = version - ver = '0' - ver = tuple([int(x) for x in ver.split(b'.')]) +def check_version(max_versions, version, arch) -> bool: + (lib, _, ver) = version.rpartition('_') + ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False - return ver <= max_versions[lib] + if isinstance(max_versions[lib], tuple): + return ver <= max_versions[lib] + else: + return ver <= max_versions[lib][arch] -def read_libraries(filename): - p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - libraries = [] - for line in stdout.split(b'\n'): - tokens = line.split() - if len(tokens)>2 and tokens[1] == b'(NEEDED)': - match = re.match(b'^Shared library: \[(.*)\]$', b' '.join(tokens[2:])) - if match: - libraries.append(match.group(1)) - else: - raise ValueError('Unparseable (NEEDED) specification') - return libraries +def check_imported_symbols(binary) -> bool: + ok: bool = True + + for symbol in binary.imported_symbols: + if not symbol.imported: + continue + + version = symbol.symbol_version if symbol.has_version else None + + if version: + aux_version = version.symbol_version_auxiliary.name if version.has_auxiliary_version else None + if aux_version and not check_version(MAX_VERSIONS, aux_version, binary.header.machine_type): + print(f'{filename}: symbol {symbol.name} from unsupported version {version}') + ok = False + return ok + +def check_exported_symbols(binary) -> bool: + ok: bool = True + + for symbol in binary.dynamic_symbols: + if not symbol.exported: + continue + name = symbol.name + if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: + continue + print(f'{binary.name}: export of symbol {name} not allowed!') + ok = False + return ok + +def check_ELF_libraries(binary) -> bool: + ok: bool = True + for library in binary.libraries: + if library not in ELF_ALLOWED_LIBRARIES: + print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_MACHO_libraries(binary) -> bool: + ok: bool = True + for dylib in binary.libraries: + split = dylib.name.split('/') + if split[-1] not in MACHO_ALLOWED_LIBRARIES: + print(f'{split[-1]} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_MACHO_min_os(binary) -> bool: + if binary.build_version.minos == [10,15,0]: + return True + return False + +def check_MACHO_sdk(binary) -> bool: + if binary.build_version.sdk == [11, 0, 0]: + return True + return False + +def check_PE_libraries(binary) -> bool: + ok: bool = True + for dylib in binary.libraries: + if dylib not in PE_ALLOWED_LIBRARIES: + print(f'{dylib} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_PE_subsystem_version(binary) -> bool: + major: int = binary.optional_header.major_subsystem_version + minor: int = binary.optional_header.minor_subsystem_version + if major == 6 and minor == 1: + return True + return False + +def check_ELF_interpreter(binary) -> bool: + expected_interpreter = ELF_INTERPRETER_NAMES[binary.header.machine_type][binary.abstract.header.endianness] + + return binary.concrete.interpreter == expected_interpreter + +CHECKS = { +lief.EXE_FORMATS.ELF: [ + ('IMPORTED_SYMBOLS', check_imported_symbols), + ('EXPORTED_SYMBOLS', check_exported_symbols), + ('LIBRARY_DEPENDENCIES', check_ELF_libraries), + ('INTERPRETER_NAME', check_ELF_interpreter), +], +lief.EXE_FORMATS.MACHO: [ + ('DYNAMIC_LIBRARIES', check_MACHO_libraries), + ('MIN_OS', check_MACHO_min_os), + ('SDK', check_MACHO_sdk), +], +lief.EXE_FORMATS.PE: [ + ('DYNAMIC_LIBRARIES', check_PE_libraries), + ('SUBSYSTEM_VERSION', check_PE_subsystem_version), +] +} if __name__ == '__main__': - cppfilt = CPPFilt() - retval = 0 + retval: int = 0 for filename in sys.argv[1:]: - # Check imported symbols - for sym,version in read_symbols(filename, True): - if version and not check_version(MAX_VERSIONS, version): - print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym).decode('utf-8'), version.decode('utf-8'))) + try: + binary = lief.parse(filename) + etype = binary.format + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') retval = 1 - # Check exported symbols - for sym,version in read_symbols(filename, False): - if sym in IGNORE_EXPORTS: continue - print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym).decode('utf-8'))) - retval = 1 - # Check dependency libraries - for library_name in read_libraries(filename): - if library_name not in ALLOWED_LIBRARIES: - print('%s: NEEDED library %s is not allowed' % (filename, library_name.decode('utf-8'))) - retval = 1 - exit(retval) + failed: List[str] = [] + for (name, func) in CHECKS[etype]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') + retval = 1 + except IOError: + print(f'{filename}: cannot open') + retval = 1 + sys.exit(retval) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 18f9835fa..d3d225f3a 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,16 +1,20 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015-2016 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Test script for security-check.py ''' -from __future__ import division,print_function +import lief #type:ignore +import os import subprocess +from typing import List import unittest +from utils import determine_wellknown_cmd + def write_testcode(filename): - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf8") as f: f.write(''' #include int main() @@ -20,43 +24,128 @@ def write_testcode(filename): } ''') +def clean_files(source, executable): + os.remove(source) + os.remove(executable) + def call_security_check(cc, source, executable, options): - subprocess.check_call([cc,source,'-o',executable] + options) - p = subprocess.Popen(['./security-check.py',executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - (stdout, stderr) = p.communicate() - return (p.returncode, stdout.rstrip()) + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) + p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + return (p.returncode, p.stdout.rstrip()) + +def get_arch(cc, source, executable): + subprocess.run([*cc, source, '-o', executable], check=True) + binary = lief.parse(executable) + arch = binary.abstract.header.architecture + os.remove(executable) + return arch class TestSecurityChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' - cc = 'gcc' + cc = determine_wellknown_cmd('CC', 'gcc') write_testcode(source) + arch = get_arch(cc, source, executable) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro']), - (1, executable+': failed PIE NX RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro']), - (1, executable+': failed PIE RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro']), - (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']), - (1, executable+': failed RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']), - (0, '')) + if arch == lief.ARCHITECTURES.X86: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE NX RELRO Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed RELRO CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code', '-fcf-protection=full']), + (0, '')) + else: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE NX RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), + (0, '')) + + clean_files(source, executable) def test_PE(self): source = 'test1.c' executable = 'test1.exe' - cc = 'i686-w64-mingw32-gcc' + cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, []), - (1, executable+': failed PIE NX')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']), + (0, '')) + + clean_files(source, executable) + + def test_MACHO(self): + source = 'test1.c' + executable = 'test1' + cc = determine_wellknown_cmd('CC', 'clang') + write_testcode(source) + arch = get_arch(cc, source, executable) + + if arch == lief.ARCHITECTURES.X86: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), + (1, executable+': failed LAZY_BINDINGS PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), + (1, executable+': failed PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), (0, '')) + else: + # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS')) + self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all']), + (1, executable+': failed LAZY_BINDINGS')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all']), + (0, '')) + + + clean_files(source, executable) if __name__ == '__main__': unittest.main() - diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py new file mode 100755 index 000000000..2881e3efa --- /dev/null +++ b/contrib/devtools/test-symbol-check.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Test script for symbol-check.py +''' +import os +import subprocess +from typing import List +import unittest + +from utils import determine_wellknown_cmd + +def call_symbol_check(cc: List[str], source, executable, options): + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) + p = subprocess.run(['./contrib/devtools/symbol-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + os.remove(source) + os.remove(executable) + return (p.returncode, p.stdout.rstrip()) + +def get_machine(cc: List[str]): + p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, universal_newlines=True) + return p.stdout.rstrip() + +class TestSymbolChecks(unittest.TestCase): + def test_ELF(self): + source = 'test1.c' + executable = 'test1' + cc = determine_wellknown_cmd('CC', 'gcc') + + # there's no way to do this test for RISC-V at the moment; we build for + # RISC-V in a glibc 2.27 environment and we allow all symbols from 2.27. + if 'riscv' in get_machine(cc): + self.skipTest("test not available for RISC-V") + + # nextup was introduced in GLIBC 2.24, so is newer than our supported + # glibc (2.18), and available in our release build environment (2.24). + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #define _GNU_SOURCE + #include + + double nextup(double x); + + int main() + { + nextup(3.14); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), + (1, executable + ': symbol nextup from unsupported version GLIBC_2.24(3)\n' + + executable + ': failed IMPORTED_SYMBOLS')) + + # -lutil is part of the libc6 package so a safe bet that it's installed + # it's also out of context enough that it's unlikely to ever become a real dependency + source = 'test2.c' + executable = 'test2' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include + + int main() + { + login(0); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lutil']), + (1, executable + ': libutil.so.1 is not in ALLOWED_LIBRARIES!\n' + + executable + ': failed LIBRARY_DEPENDENCIES')) + + # finally, check a simple conforming binary + source = 'test3.c' + executable = 'test3' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include + + int main() + { + printf("42"); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, []), + (0, '')) + + def test_MACHO(self): + source = 'test1.c' + executable = 'test1' + cc = determine_wellknown_cmd('CC', 'clang') + + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include + + int main() + { + XML_ExpatVersion(); + return 0; + } + + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lexpat', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), + (1, 'libexpat.1.dylib is not in ALLOWED_LIBRARIES!\n' + + f'{executable}: failed DYNAMIC_LIBRARIES MIN_OS SDK')) + + source = 'test2.c' + executable = 'test2' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include + + int main() + { + CGMainDisplayID(); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-framework', 'CoreGraphics', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), + (1, f'{executable}: failed MIN_OS SDK')) + + source = 'test3.c' + executable = 'test3' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + int main() + { + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,10.15', '-Wl,11.4']), + (1, f'{executable}: failed SDK')) + + def test_PE(self): + source = 'test1.c' + executable = 'test1.exe' + cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') + + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include + + int main() + { + PdhConnectMachineA(NULL); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lpdh', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), + (1, 'pdh.dll is not in ALLOWED_LIBRARIES!\n' + + executable + ': failed DYNAMIC_LIBRARIES')) + + source = 'test2.c' + executable = 'test2.exe' + + with open(source, 'w', encoding="utf8") as f: + f.write(''' + int main() + { + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,--major-subsystem-version', '-Wl,9', '-Wl,--minor-subsystem-version', '-Wl,9']), + (1, executable + ': failed SUBSYSTEM_VERSION')) + + source = 'test3.c' + executable = 'test3.exe' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include + + int main() + { + CoFreeUnusedLibrariesEx(0,0); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lole32', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), + (0, '')) + + +if __name__ == '__main__': + unittest.main() diff --git a/contrib/devtools/utils.py b/contrib/devtools/utils.py new file mode 100755 index 000000000..68ad1c3ab --- /dev/null +++ b/contrib/devtools/utils.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Common utility functions +''' +import shutil +import sys +import os +from typing import List + + +def determine_wellknown_cmd(envvar, progname) -> List[str]: + maybe_env = os.getenv(envvar) + maybe_which = shutil.which(progname) + if maybe_env: + return maybe_env.split(' ') # Well-known vars are often meant to be word-split + elif maybe_which: + return [ maybe_which ] + else: + sys.exit(f"{progname} not found") From 236fd879f8282ab86332626666110f477fee7633 Mon Sep 17 00:00:00 2001 From: Patrick Lodder Date: Thu, 12 Jan 2023 19:41:56 +0100 Subject: [PATCH 2/7] devtools: disable currently unsupported security checks Disables checks from Bitcoin 24.0.1 security-check.py code that we currently cannot support on Dogecoin Core without changes to the build process - separate-code needs linking using binutils 2.31 and/or explicit linking with -z,separate-code on binutils 2.30+ - CONTROL_FLOW can be enabled after building with gcc-8 or later. This would require at least a Ubuntu Focal Gitian implementation, and -fcf-protection enabled on the boost dependency. - HIGH_ENTROPY_VA and RELOC_SECTION checks for Windows binaries need fixes for dogecoin-cli, dogecoin-tx and test binaries, so that ASLR can be used for these binaries the same way it was done for dogecoind and dogecoin-qt. These checks can be re-enabled once these security features are enabled on release binaries (i.e. those built with Gitian) --- contrib/devtools/security-check.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 05c0af029..e4e96c30f 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -193,16 +193,25 @@ BASE_ELF = [ ('NX', check_NX), ('RELRO', check_ELF_RELRO), ('Canary', check_ELF_Canary), - ('separate_code', check_ELF_separate_code), + #('separate_code', check_ELF_separate_code), + # Note: separate_code can be enabled once release binaries are + # created with binutils 2.31 or explicitly configured on + # binutils 2.30 with -z,separate-code, + # see Bitcoin Core commit 2e9e6377 ] BASE_PE = [ ('PIE', check_PIE), ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), - ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), + #('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), + # Note: HIGH_ENTROPY_VA can be enabled when all issues with RELOC_SECTION + # are solved. ('NX', check_NX), - ('RELOC_SECTION', check_PE_RELOC_SECTION), - ('CONTROL_FLOW', check_PE_control_flow), + #('RELOC_SECTION', check_PE_RELOC_SECTION), + # Note: RELOC_SECTION is newer than our source and currently doesn't pass + # on cli tools and tests, but does work for dogecoind / dogecoin-qt + #('CONTROL_FLOW', check_PE_control_flow), + # Note: CONTROL_FLOW can be re-enabled when we build with gcc8 or higher ] BASE_MACHO = [ @@ -213,7 +222,10 @@ BASE_MACHO = [ CHECKS = { lief.EXE_FORMATS.ELF: { - lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_control_flow)], + #lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_control_flow)], + # Note: until gcc8 or higher is used for release binaries, + # do not check for CONTROL_FLOW + lief.ARCHITECTURES.X86: BASE_ELF, lief.ARCHITECTURES.ARM: BASE_ELF, lief.ARCHITECTURES.ARM64: BASE_ELF, lief.ARCHITECTURES.PPC: BASE_ELF, @@ -225,7 +237,9 @@ CHECKS = { lief.EXE_FORMATS.MACHO: { lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE), ('NX', check_NX), - ('CONTROL_FLOW', check_MACHO_control_flow)], + #('CONTROL_FLOW', check_MACHO_control_flow) + # Note: Needs change in boost for -fcf-protection + ], lief.ARCHITECTURES.ARM64: BASE_MACHO, } } @@ -260,4 +274,3 @@ if __name__ == '__main__': print(f'{filename}: cannot open') retval = 1 sys.exit(retval) - From 502f0f5c287491646f2114edc147137cc29ff7cd Mon Sep 17 00:00:00 2001 From: Patrick Lodder Date: Thu, 12 Jan 2023 19:47:35 +0100 Subject: [PATCH 3/7] devtools: reconfigure symbol-check for Dogecoin 1.14 targets Reconfigures the Bitcoin 24.0.1 symbol-check.py script to honor the maximum versions of dynamic symbols, the allowed system dependencies and allowed symbol exports. This is important to maintain when doing minor releases, because changes in these would potentially lock people out of security updates. This adds specification of the linker-loader name for i686 binaries because Bitcoin Core no longer supports that architecture. The spec was taken from: https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 Please note that: - aarch64 binaries have had a glibc 2.17 requirement since the first release with 1.14.0, and therefore have a higher glibc target than all other linux binaries. - All other values have been taken from the Dogecoin Core v1.14.6 tag, commit 3a29ba6d4. - Additional win32 and win64 needed libraries have been reverse engineered from 1.14.6 release binaries. - Windows minimum version checks have been disabled, as these need to be set on the release binaries before we check for it. --- contrib/devtools/symbol-check.py | 86 +++++++++++++++++--------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 4b1cceb57..905faa894 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -15,45 +15,55 @@ from typing import List, Dict import lief #type:ignore -# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases +# MAX_VERSIONS defines the maximum versions for dynamic symbols defined in linux +# binaries for release. These are static for each major version of Dogecoin Core. # -# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) -# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6) +# Debian 6.0.9 (Squeeze) has: # -# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases +# - g++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.11.3 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libc6) +# - libstdc++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libstdc%2B%2B6) # -# - g++ version 5.3.1 -# - libc version 2.23 +# Ubuntu 10.04.4 (Lucid Lynx) has: # -# CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product +# - g++ version 4.4.3 (http://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=lucid§ion=all) +# - libc version 2.11.1 (http://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=lucid§ion=all) +# - libstdc++ version 4.4.3 (http://packages.ubuntu.com/search?suite=lucid§ion=all&arch=any&keywords=libstdc%2B%2B&searchon=names) # -# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) -# - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) +# Taking the minimum of these as our target. # -# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. +# According to GNU ABI document (http://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: +# GCC 4.4.0: GCC_4.4.0 +# GCC 4.4.2: GLIBCXX_3.4.13, CXXABI_1.3.3 +# (glibc) GLIBC_2_11 MAX_VERSIONS = { -'GCC': (4,8,0), +'GCC': (4,4,0), 'GLIBC': { - lief.ELF.ARCH.x86_64: (2,18), - lief.ELF.ARCH.ARM: (2,18), - lief.ELF.ARCH.AARCH64:(2,18), - lief.ELF.ARCH.PPC64: (2,18), + lief.ELF.ARCH.x86_64: (2,11), + lief.ELF.ARCH.i386: (2,11), + lief.ELF.ARCH.ARM: (2,11), + lief.ELF.ARCH.AARCH64:(2,17), + lief.ELF.ARCH.PPC64: (2,17), lief.ELF.ARCH.RISCV: (2,27), }, -'LIBATOMIC': (1,0), -'V': (0,5,0), # xkb (bitcoin-qt only) +'CXXABI': (1,3,3), +'GLIBCXX': (3,4,13), +'V': (0,5,0), # xkb (dogecoin-qt only) } # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -'environ', '_environ', '__environ', '_fini', '_init', 'stdin', -'stdout', 'stderr', + '_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used', + 'stdin', 'stdout', 'stderr' } # Expected linker-loader names can be found here: # https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { + lief.ELF.ARCH.i386: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux.so.2", + }, lief.ELF.ARCH.x86_64: { lief.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", }, @@ -74,39 +84,26 @@ ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { -# bitcoind and bitcoin-qt +# dogecoind and dogecoin-qt 'libgcc_s.so.1', # GCC base support 'libc.so.6', # C library 'libpthread.so.0', # threading +'libanl.so.1', # DNS resolve 'libm.so.6', # math library 'librt.so.1', # real-time (clock) -'libatomic.so.1', 'ld-linux-x86-64.so.2', # 64-bit dynamic linker 'ld-linux.so.2', # 32-bit dynamic linker 'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker 'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker -'ld64.so.1', # POWER64 ABIv1 dynamic linker -'ld64.so.2', # POWER64 ABIv2 dynamic linker -'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker -# bitcoin-qt only +# dogecoin-qt only +'libX11-xcb.so.1', # part of X11 +'libX11.so.6', # part of X11 'libxcb.so.1', # part of X11 'libxkbcommon.so.0', # keyboard keymapping 'libxkbcommon-x11.so.0', # keyboard keymapping 'libfontconfig.so.1', # font support 'libfreetype.so.6', # font parsing -'libdl.so.2', # programming interface to dynamic linker -'libxcb-icccm.so.4', -'libxcb-image.so.0', -'libxcb-shm.so.0', -'libxcb-keysyms.so.1', -'libxcb-randr.so.0', -'libxcb-render-util.so.0', -'libxcb-render.so.0', -'libxcb-shape.so.0', -'libxcb-sync.so.1', -'libxcb-xfixes.so.0', -'libxcb-xinerama.so.0', -'libxcb-xkb.so.1', +'libdl.so.2' # programming interface to dynamic linker } MACHO_ALLOWED_LIBRARIES = { @@ -135,6 +132,7 @@ MACHO_ALLOWED_LIBRARIES = { PE_ALLOWED_LIBRARIES = { 'ADVAPI32.dll', # security & registry +'CRYPT32.dll', # crypto functions 'IPHLPAPI.DLL', # IP helper API 'KERNEL32.dll', # win32 base APIs 'msvcrt.dll', # C standard library for MSVC @@ -142,9 +140,12 @@ PE_ALLOWED_LIBRARIES = { 'USER32.dll', # user interface 'WS2_32.dll', # sockets # bitcoin-qt only +'COMDLG32.DLL', # common dialogs like "open file" +'comdlg32.dll', # same as above but lowercase in win64 binary 'dwmapi.dll', # desktop window manager 'GDI32.dll', # graphics device interface -'IMM32.dll', # input method editor +'IMM32.DLL', # input method editor +'IMM32.dll', # same as above but lowercase extension in win64 binary 'NETAPI32.dll', 'ole32.dll', # component object model 'OLEAUT32.dll', # OLE Automation API @@ -152,7 +153,9 @@ PE_ALLOWED_LIBRARIES = { 'USERENV.dll', 'UxTheme.dll', 'VERSION.dll', # version checking -'WINMM.dll', # WinMM audio API +'WINMM.DLL', # WinMM audio API +'WINMM.dll', # same as above but lowercase extension in win64 binary +'WINSPOOL.DRV', # Printer spooler driver for paper wallet printing 'WTSAPI32.dll', } @@ -256,7 +259,8 @@ lief.EXE_FORMATS.MACHO: [ ], lief.EXE_FORMATS.PE: [ ('DYNAMIC_LIBRARIES', check_PE_libraries), - ('SUBSYSTEM_VERSION', check_PE_subsystem_version), + #('SUBSYSTEM_VERSION', check_PE_subsystem_version), + #Note: needs to be set during build before we can check for it ] } From 167ca801ef72cb8a8ae7a11e21160e1ff1b494c7 Mon Sep 17 00:00:00 2001 From: Patrick Lodder Date: Fri, 13 Jan 2023 02:07:58 +0100 Subject: [PATCH 4/7] build: clean up security and symbol checks from makefile Fixes calls to make check-security and make check-symbols to have better integration with the CI and Gitian The condition in the check-symbols target requiring the configure flag --enable-glibc-back-compat is removed because that is exclusive to Linux. By removing it, we enable the check for all binaries, including those built for Windows and macOs. Finally, removes configure.ac lines for readelf and cppfilt. Those are no longer needed because lief is used in their stead. squash into: build: clean up security and symbol checks from makefile --- configure.ac | 2 -- src/Makefile.am | 6 ++---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index 60679ced0..c4cc779e1 100644 --- a/configure.ac +++ b/configure.ac @@ -86,8 +86,6 @@ AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) AC_PATH_PROG(XGETTEXT,xgettext) AC_PATH_PROG(HEXDUMP,hexdump) -AC_PATH_TOOL(READELF, readelf) -AC_PATH_TOOL(CPPFILT, c++filt) AC_PATH_TOOL(OBJCOPY, objcopy) AC_ARG_VAR(PYTHONPATH, Augments the default search path for python module files) diff --git a/src/Makefile.am b/src/Makefile.am index 61d60ce5f..69049e3a7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -507,15 +507,13 @@ clean-local: $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ check-symbols: $(bin_PROGRAMS) -if GLIBC_BACK_COMPAT @echo "Checking glibc back compat..." - $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(top_srcdir)/contrib/devtools/symbol-check.py < $(bin_PROGRAMS) -endif + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security..." - $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS) + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) endif %.pb.cc %.pb.h: %.proto From 547f0f3712c5975390a8dce418286a4f513f7154 Mon Sep 17 00:00:00 2001 From: Patrick Lodder Date: Thu, 12 Jan 2023 20:06:08 +0100 Subject: [PATCH 5/7] ci: integrate lief-based security and symbol check scripts Integrates the lief-based scripts into the GH Actions CI. This allows the CI to maintain consistent checks for an upcoming upgrade to Ubuntu focal for CI and Gitian. Because lief is not distributed as a wheel for glibc < 2.17, a custom .whl file for Ubuntu Bionic is made available on depends.dogecoincore.org to save up to an hour that would otherwise be spent on compiling lief from source. For current focal-based CI jobs, this is not needed because that provides glibc > 2.17. Each CI job has received 2 extra steps that are mutually exclusive to make sure that the correct version is installed. When there are no longer any Ubuntu Bionic based bionic jobs, this can be deleted in favor of a single command in the "install packages" step. python3-pip and python3-setuptools are now installed by default for all CI jobs, where before this was only used for jobs that ran the full qa test suite. --- .github/workflows/ci.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f512838e..17a434d6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: CACHE_NONCE: "1" WINEDEBUG: fixme-all SDK_URL: https://depends.dogecoincore.org + BIONIC_LIEF_WHL: lief-0.12.3-cp36-cp36m-linux_x86_64.whl strategy: fail-fast: false @@ -45,7 +46,7 @@ jobs: - name: i686-linux host: i686-pc-linux-gnu os: ubuntu-18.04 - packages: g++-multilib bc python3-pip python3-setuptools python3-zmq + packages: g++-multilib bc python3-zmq run-bench: true test-script: | make check $MAKEJOBS VERBOSE=1 @@ -119,7 +120,7 @@ jobs: - name: x86_64-linux-dbg host: x86_64-unknown-linux-gnu os: ubuntu-18.04 - packages: bc python3-pip python3-setuptools python3-zmq + packages: bc python3-zmq run-bench: true test-script: | make check $MAKEJOBS VERBOSE=1 @@ -187,7 +188,7 @@ jobs: - name: x86_64-macos host: x86_64-apple-darwin11 os: ubuntu-18.04 - packages: cmake imagemagick libcap-dev librsvg2-bin libz-dev libtiff-tools libtinfo5 python3-setuptools xorriso + packages: cmake imagemagick libcap-dev librsvg2-bin libz-dev libtiff-tools libtinfo5 xorriso run-bench: false check-security: false check-symbols: false @@ -199,7 +200,7 @@ jobs: - name: x86_64-linux-experimental host: x86_64-linux-gnu os: ubuntu-18.04 - packages: bc python3-pip python3-setuptools python3-zmq + packages: bc python3-zmq run-bench: true test-script: | make check $MAKEJOBS VERBOSE=1 @@ -220,8 +221,19 @@ jobs: - name: Install packages run: | sudo apt-get update - sudo apt-get install build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison + sudo apt-get install build-essential libtool autotools-dev automake \ + pkg-config bsdmainutils curl ca-certificates ccache rsync git \ + procps bison python3 python3-pip python3-setuptools python3-wheel sudo apt-get install ${{ matrix.packages }} + python3 -m pip install setuptools --upgrade + + - name: Install custom lief wheel + if: matrix.os == 'ubuntu-18.04' + run: python3 -m pip install $SDK_URL/$BIONIC_LIEF_WHL + + - name: Install standard lief wheel + if: matrix.os != 'ubuntu-18.04' + run: python3 -m pip install lief - name: Post install if: ${{ matrix.postinstall }} From a680438a24e9540d9dba9831021298f47f1c8efe Mon Sep 17 00:00:00 2001 From: Patrick Lodder Date: Fri, 13 Jan 2023 00:30:57 +0100 Subject: [PATCH 6/7] ci: enable security and symbol checks for all supported targets This enables running of security checks for macOS and symbol checks for windows and ARM linux targets with each CI run. Symbol checks remain disabled for debug and experimental targets because those aren't production binary releases. macOS symbol checks need build system patches before it will work as intended so these have to stay disabled at this point. --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17a434d6f..7fc2f0405 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ jobs: test-script: | qemu-arm -E LD_LIBRARY_PATH=/usr/arm-linux-gnueabihf/lib/ /usr/arm-linux-gnueabihf/lib/ld-linux-armhf.so.3 src/test/test_dogecoin check-security: true - check-symbols: false + check-symbols: true dep-opts: "NO_QT=1" config-opts: "--enable-glibc-back-compat LDFLAGS=-static-libstdc++" goal: install @@ -101,7 +101,7 @@ jobs: test-script: | qemu-aarch64 -E LD_LIBRARY_PATH=/usr/aarch64-linux-gnu/lib/ /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 src/test/test_dogecoin check-security: true - check-symbols: false + check-symbols: true dep-opts: "NO_QT=1" config-opts: "--enable-zmq --enable-glibc-back-compat LDFLAGS=-static-libstdc++" goal: install @@ -145,7 +145,7 @@ jobs: test-script: | make check $MAKEJOBS VERBOSE=1 check-security: true - check-symbols: false + check-symbols: true dep-opts: "" config-opts: "--enable-gui=qt5" goal: install @@ -163,7 +163,7 @@ jobs: test-script: | make check $MAKEJOBS VERBOSE=1 check-security: true - check-symbols: false + check-symbols: true dep-opts: "" config-opts: "--enable-gui=qt5" goal: install @@ -190,7 +190,7 @@ jobs: os: ubuntu-18.04 packages: cmake imagemagick libcap-dev librsvg2-bin libz-dev libtiff-tools libtinfo5 xorriso run-bench: false - check-security: false + check-security: true check-symbols: false dep-opts: "" config-opts: "--with-gui=qt5 --disable-tests" From a77bcf0f46fcaaf504f468a285d281877964d671 Mon Sep 17 00:00:00 2001 From: Patrick Lodder Date: Mon, 16 Jan 2023 19:07:07 +0100 Subject: [PATCH 7/7] build: integrate lief-based checks into gitian descriptors Integrates modernized security and symbol checks into all bionic gitian descriptors - uses the precompiled bionic-specific lief wheel from depends.dogecoincore.org to save an hour build time on each gitian host. This does require pre-downloading the wheel file like done for the osx SDK. - replaces python2 with python3 in descriptors - adds python3-setuptools and python3-pip - now requires lief-0.12.3-cp36-cp36m-linux_x86_64.whl to be present in the gitian-builder/inputs folder - installs the wheel prior to installing dependencies - enables symbol check for windows - enables security check for osx - adds automatic wheel download to gitian-build.sh --- contrib/gitian-build.sh | 5 +++++ contrib/gitian-descriptors/gitian-linux.yml | 21 +++++++++------------ contrib/gitian-descriptors/gitian-osx.yml | 7 +++++++ contrib/gitian-descriptors/gitian-win.yml | 11 +++++++++-- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/contrib/gitian-build.sh b/contrib/gitian-build.sh index 254ccdf13..a0f6aca0b 100755 --- a/contrib/gitian-build.sh +++ b/contrib/gitian-build.sh @@ -23,6 +23,10 @@ ossTarHash="f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9" macosSdkUrl="https://depends.dogecoincore.org/MacOSX10.11.sdk.tar.gz" macosSdkHash="bec9d089ebf2e2dd59b1a811a38ec78ebd5da18cbbcd6ab39d1e59f64ac5033f" +# lief custom wheel is only needed for bionic-based gitian +lief="https://depends.dogecoincore.org/lief-0.12.3-cp36-cp36m-linux_x86_64.whl" +liefHash="c84cbdb32c8a830fbb82c907b733050c7fc5c9bf4f51a46541f1b8c2e48def9f" + # What to do verify=false build=false @@ -267,6 +271,7 @@ if [[ $setup == true ]]; then download_file $ossPatchUrl $ossPatchHash download_file $ossTarUrl $ossTarHash download_file $macosSdkUrl $macosSdkHash + download_file $lief $liefHash popd diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 4458bb98a..c270da052 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -27,11 +27,14 @@ packages: - "bison" - "bsdmainutils" - "ca-certificates" -- "python" +- "python3" +- "python3-setuptools" +- "python3-pip" remotes: - "url": "https://github.com/dogecoin/dogecoin.git" "dir": "dogecoin" -files: [] +files: +- "lief-0.12.3-cp36-cp36m-linux_x86_64.whl" script: | WRAP_DIR=$HOME/wrapped @@ -114,6 +117,9 @@ script: | chmod +x ${WRAP_DIR}/${prog} done + # install python-lief + python3 -m pip install ${BUILD_DIR}/lief-0.12.3-cp36-cp36m-linux_x86_64.whl + cd dogecoin BASEPREFIX=`pwd`/depends # Build dependencies for each host @@ -158,16 +164,7 @@ script: | CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security - - #TODO: This is a quick hack that disables symbol checking for arm. - # Instead, we should investigate why these are popping up. - # For aarch64, we'll need to bump up the min GLIBC version, as the abi - # support wasn't introduced until 2.17. - case $i in - aarch64-*) : ;; - arm-*) : ;; - *) make ${MAKEOPTS} -C src check-symbols ;; - esac + make ${MAKEOPTS} -C src check-symbols make install DESTDIR=${INSTALLPATH} cd installed diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 31259de25..224dcecba 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -26,12 +26,14 @@ packages: - "python3" - "python3-dev" - "python3-setuptools" +- "python3-pip" - "fonts-tuffy" remotes: - "url": "https://github.com/dogecoin/dogecoin.git" "dir": "dogecoin" files: - "MacOSX10.11.sdk.tar.gz" +- "lief-0.12.3-cp36-cp36m-linux_x86_64.whl" script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin11" @@ -89,6 +91,9 @@ script: | mkdir -p ${BASEPREFIX}/SDKs tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz + # install python-lief + python3 -m pip install ${BUILD_DIR}/lief-0.12.3-cp36-cp36m-linux_x86_64.whl + # Build dependencies for each host for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" @@ -103,6 +108,8 @@ script: | # Create the release tarball using (arbitrarily) the first host ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ + make ${MAKEOPTS} + make ${MAKEOPTS} -C src check-security make dist SOURCEDIST=`echo dogecoin-*.tar.gz` DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 1b691c519..3a1d88fdb 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -20,12 +20,15 @@ packages: - "nsis" - "zip" - "ca-certificates" -- "python" +- "python3" +- "python3-setuptools" +- "python3-pip" - "rename" remotes: - "url": "https://github.com/dogecoin/dogecoin.git" "dir": "dogecoin" -files: [] +files: +- "lief-0.12.3-cp36-cp36m-linux_x86_64.whl" script: | WRAP_DIR=$HOME/wrapped HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" @@ -94,6 +97,9 @@ script: | create_per-host_compiler_wrapper "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} + # install python-lief + python3 -m pip install ${BUILD_DIR}/lief-0.12.3-cp36-cp36m-linux_x86_64.whl + cd dogecoin BASEPREFIX=`pwd`/depends # Build dependencies for each host @@ -137,6 +143,7 @@ script: | CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security + make ${MAKEOPTS} -C src check-symbols make deploy make install DESTDIR=${INSTALLPATH} cp -f dogecoin-*setup*.exe $OUTDIR/