[1/4] ca-certificates: Update to work with python3 version of certdata2pem.py
Commit Message
- Implement python3 version of certdata2pem.py script from fedora
- Modify build.sh to work with python3 script that uses p11-kit based on fedora
approach - https://src.fedoraproject.org/rpms/ca-certificates/tree/rawhide
- Extraction of cert files now uses p11-kit which requires libtasn1 as a build
dependency
- Updated rootfile
- Updated ca-certificates installed into a vm and confirmed to download a file from an
https site with the same results as with existing ca-certfictaes system
Tested-by: Adolf Belka <adolf.belka@ipfire.org>
Signed-off-by: Adolf Belka <adolf.belka@ipfire.org>
---
config/ca-certificates/build.sh | 48 +++--
config/ca-certificates/certdata2pem.py | 260 ++++++++++++++++++++----
config/rootfiles/common/ca-certificates | 5 +-
lfs/ca-certificates | 2 +-
4 files changed, 248 insertions(+), 67 deletions(-)
Comments
Reviewed-by: Michael Tremer <michael.tremer@ipfire.org>
> On 20 Aug 2021, at 21:04, Adolf Belka <adolf.belka@ipfire.org> wrote:
>
> - Implement python3 version of certdata2pem.py script from fedora
> - Modify build.sh to work with python3 script that uses p11-kit based on fedora
> approach - https://src.fedoraproject.org/rpms/ca-certificates/tree/rawhide
> - Extraction of cert files now uses p11-kit which requires libtasn1 as a build
> dependency
> - Updated rootfile
> - Updated ca-certificates installed into a vm and confirmed to download a file from an
> https site with the same results as with existing ca-certfictaes system
>
> Tested-by: Adolf Belka <adolf.belka@ipfire.org>
> Signed-off-by: Adolf Belka <adolf.belka@ipfire.org>
> ---
> config/ca-certificates/build.sh | 48 +++--
> config/ca-certificates/certdata2pem.py | 260 ++++++++++++++++++++----
> config/rootfiles/common/ca-certificates | 5 +-
> lfs/ca-certificates | 2 +-
> 4 files changed, 248 insertions(+), 67 deletions(-)
>
> diff --git a/config/ca-certificates/build.sh b/config/ca-certificates/build.sh
> index c868ed94a..8e64f9e9f 100644
> --- a/config/ca-certificates/build.sh
> +++ b/config/ca-certificates/build.sh
> @@ -3,13 +3,34 @@
> set -e
>
> # Create file layout.
> -mkdir -pv certs certs/legacy-default certs/legacy-disable
> +mkdir -pv certs
> +mkdir -pv /etc/pki/ca-trust/source
> cp certdata.txt certs
> cd certs
>
> -python ../certdata2pem.py
> +python3 ../certdata2pem.py
>
> cd ..
> +
> +
> +cat <<EOF > ca-bundle.trust.p11-kit
> +# This is a bundle of X.509 certificates of public Certificate
> +# Authorities. It was generated from the Mozilla root CA list.
> +# These certificates and trust/distrust attributes use the file format accepted
> +# by the p11-kit-trust module.
> +#
> +# Source: mozilla/security/nss/lib/ckfw/builtins/certdata.txt
> +#
> +EOF
> +
> +
> +P11FILES=`find certs -name \*.tmp-p11-kit | wc -l`
> +if [ $P11FILES -ne 0 ]; then
> + for p in certs/*.tmp-p11-kit; do
> + cat "$p" >> /etc/pki/ca-trust/source/ca-bundle.trust.p11-kit
> + done
> +fi
> +
> cat <<EOF > ca-bundle.crt
> # This is a bundle of X.509 certificates of public Certificate
> # Authorities. It was generated from the Mozilla root CA list.
> @@ -28,24 +49,11 @@ cat <<EOF > ca-bundle.trust.crt
> #
> EOF
>
> -for f in certs/*.crt; do
> - [ -z "${f}" ] && continue
> -
> - tbits=$(sed -n '/^# openssl-trust/{s/^.*=//;p;}' ${f})
> - case "${tbits}" in
> - *serverAuth*)
> - openssl x509 -text -in "${f}" >> ca-bundle.crt
> - ;;
> - esac
> +trust extract --comment --filter=certificates --format=openssl-bundle --overwrite ca-bundle.trust
> +cat ca-bundle.trust >> ca-bundle.trust.crt
>
> - if [ -n "$tbits" ]; then
> - targs=""
> - for t in ${tbits}; do
> - targs="${targs} -addtrust ${t}"
> - done
> +trust extract --comment --filter=ca-anchors --format=pem-bundle --overwrite --purpose server-auth ca-bundle
> +cat ca-bundle >> ca-bundle.crt
>
> - openssl x509 -text -in "${f}" -trustout $targs >> ca-bundle.trust.crt
> - fi
> -done
>
> -exit 0
> +exit 0
> \ No newline at end of file
> diff --git a/config/ca-certificates/certdata2pem.py b/config/ca-certificates/certdata2pem.py
> index 44cc9e03b..a52ce9c74 100644
> --- a/config/ca-certificates/certdata2pem.py
> +++ b/config/ca-certificates/certdata2pem.py
> @@ -26,16 +26,17 @@ import os.path
> import re
> import sys
> import textwrap
> -import urllib
> +import urllib.request, urllib.parse, urllib.error
> +import subprocess
>
> objects = []
>
> def printable_serial(obj):
> - return ".".join(map(lambda x:str(ord(x)), obj['CKA_SERIAL_NUMBER']))
> + return ".".join([str(x) for x in obj['CKA_SERIAL_NUMBER']])
>
> # Dirty file parser.
> in_data, in_multiline, in_obj = False, False, False
> -field, type, value, obj = None, None, None, dict()
> +field, ftype, value, binval, obj = None, None, None, bytearray(), dict()
> for line in open('certdata.txt', 'r'):
> # Ignore the file header.
> if not in_data:
> @@ -55,33 +56,36 @@ for line in open('certdata.txt', 'r'):
> continue
> if in_multiline:
> if not line.startswith('END'):
> - if type == 'MULTILINE_OCTAL':
> + if ftype == 'MULTILINE_OCTAL':
> line = line.strip()
> for i in re.finditer(r'\\([0-3][0-7][0-7])', line):
> - value += chr(int(i.group(1), 8))
> + integ = int(i.group(1), 8)
> + binval.extend((integ).to_bytes(1, sys.byteorder))
> + obj[field] = binval
> else:
> value += line
> + obj[field] = value
> continue
> - obj[field] = value
> in_multiline = False
> continue
> if line.startswith('CKA_CLASS'):
> in_obj = True
> line_parts = line.strip().split(' ', 2)
> if len(line_parts) > 2:
> - field, type = line_parts[0:2]
> + field, ftype = line_parts[0:2]
> value = ' '.join(line_parts[2:])
> elif len(line_parts) == 2:
> - field, type = line_parts
> + field, ftype = line_parts
> value = None
> else:
> - raise NotImplementedError, 'line_parts < 2 not supported.\n' + line
> - if type == 'MULTILINE_OCTAL':
> + raise NotImplementedError('line_parts < 2 not supported.\n' + line)
> + if ftype == 'MULTILINE_OCTAL':
> in_multiline = True
> value = ""
> + binval = bytearray()
> continue
> obj[field] = value
> -if len(obj.items()) > 0:
> +if len(list(obj.items())) > 0:
> objects.append(obj)
>
> # Build up trust database.
> @@ -91,7 +95,7 @@ for obj in objects:
> continue
> key = obj['CKA_LABEL'] + printable_serial(obj)
> trustmap[key] = obj
> - print " added trust", key
> + print(" added trust", key)
>
> # Build up cert database.
> certmap = dict()
> @@ -100,7 +104,7 @@ for obj in objects:
> continue
> key = obj['CKA_LABEL'] + printable_serial(obj)
> certmap[key] = obj
> - print " added cert", key
> + print(" added cert", key)
>
> def obj_to_filename(obj):
> label = obj['CKA_LABEL'][1:-1]
> @@ -109,10 +113,32 @@ def obj_to_filename(obj):
> .replace('(', '=')\
> .replace(')', '=')\
> .replace(',', '_')
> - label = re.sub(r'\\x[0-9a-fA-F]{2}', lambda m:chr(int(m.group(0)[2:], 16)), label)
> + labelbytes = bytearray()
> + i = 0
> + imax = len(label)
> + while i < imax:
> + if i < imax-3 and label[i] == '\\' and label[i+1] == 'x':
> + labelbytes.extend(bytes.fromhex(label[i+2:i+4]))
> + i += 4
> + continue
> + labelbytes.extend(str.encode(label[i]))
> + i = i+1
> + continue
> + label = labelbytes.decode('utf-8')
> serial = printable_serial(obj)
> return label + ":" + serial
>
> +def write_cert_ext_to_file(f, oid, value, public_key):
> + f.write("[p11-kit-object-v1]\n")
> + f.write("label: ");
> + f.write(tobj['CKA_LABEL'])
> + f.write("\n")
> + f.write("class: x-certificate-extension\n");
> + f.write("object-id: " + oid + "\n")
> + f.write("value: \"" + value + "\"\n")
> + f.write("modifiable: false\n");
> + f.write(public_key)
> +
> trust_types = {
> "CKA_TRUST_DIGITAL_SIGNATURE": "digital-signature",
> "CKA_TRUST_NON_REPUDIATION": "non-repudiation",
> @@ -151,34 +177,39 @@ openssl_trust = {
> "CKA_TRUST_EMAIL_PROTECTION": "emailProtection",
> }
>
> +cert_distrust_types = {
> + "CKA_NSS_SERVER_DISTRUST_AFTER": "nss-server-distrust-after",
> + "CKA_NSS_EMAIL_DISTRUST_AFTER": "nss-email-distrust-after",
> +}
> +
> for tobj in objects:
> if tobj['CKA_CLASS'] == 'CKO_NSS_TRUST':
> key = tobj['CKA_LABEL'] + printable_serial(tobj)
> - print "producing trust for " + key
> + print("producing trust for " + key)
> trustbits = []
> distrustbits = []
> openssl_trustflags = []
> openssl_distrustflags = []
> legacy_trustbits = []
> legacy_openssl_trustflags = []
> - for t in trust_types.keys():
> - if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
> + for t in list(trust_types.keys()):
> + if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
> trustbits.append(t)
> if t in openssl_trust:
> openssl_trustflags.append(openssl_trust[t])
> - if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
> + if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
> distrustbits.append(t)
> if t in openssl_trust:
> openssl_distrustflags.append(openssl_trust[t])
>
> - for t in legacy_trust_types.keys():
> - if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
> + for t in list(legacy_trust_types.keys()):
> + if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
> real_t = legacy_to_real_trust_types[t]
> legacy_trustbits.append(real_t)
> if real_t in openssl_trust:
> legacy_openssl_trustflags.append(openssl_trust[real_t])
> - if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
> - raise NotImplementedError, 'legacy distrust not supported.\n' + line
> + if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
> + raise NotImplementedError('legacy distrust not supported.\n' + line)
>
> fname = obj_to_filename(tobj)
> try:
> @@ -186,43 +217,181 @@ for tobj in objects:
> except:
> obj = None
>
> - if obj != None:
> - fname += ".crt"
> - else:
> - fname += ".p11-kit"
> + # optional debug code, that dumps the parsed input to files
> + #fulldump = "dump-" + fname
> + #dumpf = open(fulldump, 'w')
> + #dumpf.write(str(obj));
> + #dumpf.write(str(tobj));
> + #dumpf.close();
>
> is_legacy = 0
> - if tobj.has_key('LEGACY_CKA_TRUST_SERVER_AUTH') or tobj.has_key('LEGACY_CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('LEGACY_CKA_TRUST_CODE_SIGNING'):
> + if 'LEGACY_CKA_TRUST_SERVER_AUTH' in tobj or 'LEGACY_CKA_TRUST_EMAIL_PROTECTION' in tobj or 'LEGACY_CKA_TRUST_CODE_SIGNING' in tobj:
> is_legacy = 1
> if obj == None:
> - raise NotImplementedError, 'found legacy trust without certificate.\n' + line
> - legacy_fname = "legacy-default/" + fname
> + raise NotImplementedError('found legacy trust without certificate.\n' + line)
> +
> + legacy_fname = "legacy-default/" + fname + ".crt"
> f = open(legacy_fname, 'w')
> f.write("# alias=%s\n"%tobj['CKA_LABEL'])
> f.write("# trust=" + " ".join(legacy_trustbits) + "\n")
> if legacy_openssl_trustflags:
> f.write("# openssl-trust=" + " ".join(legacy_openssl_trustflags) + "\n")
> f.write("-----BEGIN CERTIFICATE-----\n")
> - f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
> + temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
> + temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
> + f.write("\n".join(temp_wrapped))
> f.write("\n-----END CERTIFICATE-----\n")
> f.close()
> - if tobj.has_key('CKA_TRUST_SERVER_AUTH') or tobj.has_key('CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('CKA_TRUST_CODE_SIGNING'):
> - fname = "legacy-disable/" + fname
> - else:
> - continue
>
> + if 'CKA_TRUST_SERVER_AUTH' in tobj or 'CKA_TRUST_EMAIL_PROTECTION' in tobj or 'CKA_TRUST_CODE_SIGNING' in tobj:
> + legacy_fname = "legacy-disable/" + fname + ".crt"
> + f = open(legacy_fname, 'w')
> + f.write("# alias=%s\n"%tobj['CKA_LABEL'])
> + f.write("# trust=" + " ".join(trustbits) + "\n")
> + if openssl_trustflags:
> + f.write("# openssl-trust=" + " ".join(openssl_trustflags) + "\n")
> + f.write("-----BEGIN CERTIFICATE-----\n")
> + f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
> + f.write("\n-----END CERTIFICATE-----\n")
> + f.close()
> +
> + # don't produce p11-kit output for legacy certificates
> + continue
> +
> + pk = ''
> + cert_comment = ''
> + if obj != None:
> + # must extract the public key from the cert, let's use openssl
> + cert_fname = "cert-" + fname
> + fc = open(cert_fname, 'w')
> + fc.write("-----BEGIN CERTIFICATE-----\n")
> + temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
> + temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
> + fc.write("\n".join(temp_wrapped))
> + fc.write("\n-----END CERTIFICATE-----\n")
> + fc.close();
> + pk_fname = "pubkey-" + fname
> + fpkout = open(pk_fname, "w")
> + dump_pk_command = ["openssl", "x509", "-in", cert_fname, "-noout", "-pubkey"]
> + subprocess.call(dump_pk_command, stdout=fpkout)
> + fpkout.close()
> + with open (pk_fname, "r") as myfile:
> + pk=myfile.read()
> + # obtain certificate information suitable as a comment
> + comment_fname = "comment-" + fname
> + fcout = open(comment_fname, "w")
> + comment_command = ["openssl", "x509", "-in", cert_fname, "-noout", "-text"]
> + subprocess.call(comment_command, stdout=fcout)
> + fcout.close()
> + sed_command = ["sed", "--in-place", "s/^/#/", comment_fname]
> + subprocess.call(sed_command)
> + with open (comment_fname, "r", errors = 'replace') as myfile:
> + cert_comment=myfile.read()
> +
> + fname += ".tmp-p11-kit"
> f = open(fname, 'w')
> +
> if obj != None:
> - f.write("# alias=%s\n"%tobj['CKA_LABEL'])
> - f.write("# trust=" + " ".join(trustbits) + "\n")
> - f.write("# distrust=" + " ".join(distrustbits) + "\n")
> - if openssl_trustflags:
> - f.write("# openssl-trust=" + " ".join(openssl_trustflags) + "\n")
> - if openssl_distrustflags:
> - f.write("# openssl-distrust=" + " ".join(openssl_distrustflags) + "\n")
> + is_distrusted = False
> + has_server_trust = False
> + has_email_trust = False
> + has_code_trust = False
> +
> + if 'CKA_TRUST_SERVER_AUTH' in tobj:
> + if tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED':
> + is_distrusted = True
> + elif tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_TRUSTED_DELEGATOR':
> + has_server_trust = True
> +
> + if 'CKA_TRUST_EMAIL_PROTECTION' in tobj:
> + if tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED':
> + is_distrusted = True
> + elif tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_TRUSTED_DELEGATOR':
> + has_email_trust = True
> +
> + if 'CKA_TRUST_CODE_SIGNING' in tobj:
> + if tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED':
> + is_distrusted = True
> + elif tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_TRUSTED_DELEGATOR':
> + has_code_trust = True
> +
> + if is_distrusted:
> + trust_ext_oid = "1.3.6.1.4.1.3319.6.10.1"
> + trust_ext_value = "0.%06%0a%2b%06%01%04%01%99w%06%0a%01%04 0%1e%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
> + write_cert_ext_to_file(f, trust_ext_oid, trust_ext_value, pk)
> +
> + trust_ext_oid = "2.5.29.37"
> + if has_server_trust:
> + if has_email_trust:
> + if has_code_trust:
> + # server + email + code
> + trust_ext_value = "0%2a%06%03U%1d%25%01%01%ff%04 0%1e%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
> + else:
> + # server + email
> + trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01"
> + else:
> + if has_code_trust:
> + # server + code
> + trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
> + else:
> + # server
> + trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%01"
> + else:
> + if has_email_trust:
> + if has_code_trust:
> + # email + code
> + trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%03"
> + else:
> + # email
> + trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%04"
> + else:
> + if has_code_trust:
> + # code
> + trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%03"
> + else:
> + # none
> + trust_ext_value = "0%18%06%03U%1d%25%01%01%ff%04%0e0%0c%06%0a%2b%06%01%04%01%99w%06%0a%10"
> +
> + # no 2.5.29.37 for neutral certificates
> + if (is_distrusted or has_server_trust or has_email_trust or has_code_trust):
> + write_cert_ext_to_file(f, trust_ext_oid, trust_ext_value, pk)
> +
> + pk = ''
> + f.write("\n")
> +
> + f.write("[p11-kit-object-v1]\n")
> + f.write("label: ");
> + f.write(tobj['CKA_LABEL'])
> + f.write("\n")
> + if is_distrusted:
> + f.write("x-distrusted: true\n")
> + elif has_server_trust or has_email_trust or has_code_trust:
> + f.write("trusted: true\n")
> + else:
> + f.write("trusted: false\n")
> +
> + # requires p11-kit >= 0.23.4
> + f.write("nss-mozilla-ca-policy: true\n")
> + f.write("modifiable: false\n");
> +
> + # requires p11-kit >= 0.23.19
> + for t in list(cert_distrust_types.keys()):
> + if t in obj:
> + value = obj[t]
> + if value == 'CK_FALSE':
> + value = bytearray(1)
> + f.write(cert_distrust_types[t] + ": \"")
> + f.write(urllib.parse.quote(value));
> + f.write("\"\n")
> +
> f.write("-----BEGIN CERTIFICATE-----\n")
> - f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
> + temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
> + temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
> + f.write("\n".join(temp_wrapped))
> f.write("\n-----END CERTIFICATE-----\n")
> + f.write(cert_comment)
> + f.write("\n")
> +
> else:
> f.write("[p11-kit-object-v1]\n")
> f.write("label: ");
> @@ -230,14 +399,15 @@ for tobj in objects:
> f.write("\n")
> f.write("class: certificate\n")
> f.write("certificate-type: x-509\n")
> + f.write("modifiable: false\n");
> f.write("issuer: \"");
> - f.write(urllib.quote(tobj['CKA_ISSUER']));
> + f.write(urllib.parse.quote(tobj['CKA_ISSUER']));
> f.write("\"\n")
> f.write("serial-number: \"");
> - f.write(urllib.quote(tobj['CKA_SERIAL_NUMBER']));
> + f.write(urllib.parse.quote(tobj['CKA_SERIAL_NUMBER']));
> f.write("\"\n")
> if (tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED'):
> f.write("x-distrusted: true\n")
> f.write("\n\n")
> f.close()
> - print " -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags)
> + print(" -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags))
> diff --git a/config/rootfiles/common/ca-certificates b/config/rootfiles/common/ca-certificates
> index 087c3e450..06eb66f3b 100644
> --- a/config/rootfiles/common/ca-certificates
> +++ b/config/rootfiles/common/ca-certificates
> @@ -1,4 +1,7 @@
> +#etc/pki
> +#etc/pki/ca-trust
> +#etc/pki/ca-trust/source
> +etc/pki/ca-trust/source/ca-bundle.trust.p11-kit
> etc/ssl/cert.pem
> -#etc/ssl/certs
> etc/ssl/certs/ca-bundle.crt
> etc/ssl/certs/ca-bundle.trust.crt
> diff --git a/lfs/ca-certificates b/lfs/ca-certificates
> index f3c68a7c0..9e37687da 100644
> --- a/lfs/ca-certificates
> +++ b/lfs/ca-certificates
> @@ -24,7 +24,7 @@
>
> include Config
>
> -VER = 20210611
> +VER = 20210819
>
> THISAPP = ca-certificates
> DIR_APP = $(DIR_SRC)/$(THISAPP)
> --
> 2.33.0
>
@@ -3,13 +3,34 @@
set -e
# Create file layout.
-mkdir -pv certs certs/legacy-default certs/legacy-disable
+mkdir -pv certs
+mkdir -pv /etc/pki/ca-trust/source
cp certdata.txt certs
cd certs
-python ../certdata2pem.py
+python3 ../certdata2pem.py
cd ..
+
+
+cat <<EOF > ca-bundle.trust.p11-kit
+# This is a bundle of X.509 certificates of public Certificate
+# Authorities. It was generated from the Mozilla root CA list.
+# These certificates and trust/distrust attributes use the file format accepted
+# by the p11-kit-trust module.
+#
+# Source: mozilla/security/nss/lib/ckfw/builtins/certdata.txt
+#
+EOF
+
+
+P11FILES=`find certs -name \*.tmp-p11-kit | wc -l`
+if [ $P11FILES -ne 0 ]; then
+ for p in certs/*.tmp-p11-kit; do
+ cat "$p" >> /etc/pki/ca-trust/source/ca-bundle.trust.p11-kit
+ done
+fi
+
cat <<EOF > ca-bundle.crt
# This is a bundle of X.509 certificates of public Certificate
# Authorities. It was generated from the Mozilla root CA list.
@@ -28,24 +49,11 @@ cat <<EOF > ca-bundle.trust.crt
#
EOF
-for f in certs/*.crt; do
- [ -z "${f}" ] && continue
-
- tbits=$(sed -n '/^# openssl-trust/{s/^.*=//;p;}' ${f})
- case "${tbits}" in
- *serverAuth*)
- openssl x509 -text -in "${f}" >> ca-bundle.crt
- ;;
- esac
+trust extract --comment --filter=certificates --format=openssl-bundle --overwrite ca-bundle.trust
+cat ca-bundle.trust >> ca-bundle.trust.crt
- if [ -n "$tbits" ]; then
- targs=""
- for t in ${tbits}; do
- targs="${targs} -addtrust ${t}"
- done
+trust extract --comment --filter=ca-anchors --format=pem-bundle --overwrite --purpose server-auth ca-bundle
+cat ca-bundle >> ca-bundle.crt
- openssl x509 -text -in "${f}" -trustout $targs >> ca-bundle.trust.crt
- fi
-done
-exit 0
+exit 0
\ No newline at end of file
@@ -26,16 +26,17 @@ import os.path
import re
import sys
import textwrap
-import urllib
+import urllib.request, urllib.parse, urllib.error
+import subprocess
objects = []
def printable_serial(obj):
- return ".".join(map(lambda x:str(ord(x)), obj['CKA_SERIAL_NUMBER']))
+ return ".".join([str(x) for x in obj['CKA_SERIAL_NUMBER']])
# Dirty file parser.
in_data, in_multiline, in_obj = False, False, False
-field, type, value, obj = None, None, None, dict()
+field, ftype, value, binval, obj = None, None, None, bytearray(), dict()
for line in open('certdata.txt', 'r'):
# Ignore the file header.
if not in_data:
@@ -55,33 +56,36 @@ for line in open('certdata.txt', 'r'):
continue
if in_multiline:
if not line.startswith('END'):
- if type == 'MULTILINE_OCTAL':
+ if ftype == 'MULTILINE_OCTAL':
line = line.strip()
for i in re.finditer(r'\\([0-3][0-7][0-7])', line):
- value += chr(int(i.group(1), 8))
+ integ = int(i.group(1), 8)
+ binval.extend((integ).to_bytes(1, sys.byteorder))
+ obj[field] = binval
else:
value += line
+ obj[field] = value
continue
- obj[field] = value
in_multiline = False
continue
if line.startswith('CKA_CLASS'):
in_obj = True
line_parts = line.strip().split(' ', 2)
if len(line_parts) > 2:
- field, type = line_parts[0:2]
+ field, ftype = line_parts[0:2]
value = ' '.join(line_parts[2:])
elif len(line_parts) == 2:
- field, type = line_parts
+ field, ftype = line_parts
value = None
else:
- raise NotImplementedError, 'line_parts < 2 not supported.\n' + line
- if type == 'MULTILINE_OCTAL':
+ raise NotImplementedError('line_parts < 2 not supported.\n' + line)
+ if ftype == 'MULTILINE_OCTAL':
in_multiline = True
value = ""
+ binval = bytearray()
continue
obj[field] = value
-if len(obj.items()) > 0:
+if len(list(obj.items())) > 0:
objects.append(obj)
# Build up trust database.
@@ -91,7 +95,7 @@ for obj in objects:
continue
key = obj['CKA_LABEL'] + printable_serial(obj)
trustmap[key] = obj
- print " added trust", key
+ print(" added trust", key)
# Build up cert database.
certmap = dict()
@@ -100,7 +104,7 @@ for obj in objects:
continue
key = obj['CKA_LABEL'] + printable_serial(obj)
certmap[key] = obj
- print " added cert", key
+ print(" added cert", key)
def obj_to_filename(obj):
label = obj['CKA_LABEL'][1:-1]
@@ -109,10 +113,32 @@ def obj_to_filename(obj):
.replace('(', '=')\
.replace(')', '=')\
.replace(',', '_')
- label = re.sub(r'\\x[0-9a-fA-F]{2}', lambda m:chr(int(m.group(0)[2:], 16)), label)
+ labelbytes = bytearray()
+ i = 0
+ imax = len(label)
+ while i < imax:
+ if i < imax-3 and label[i] == '\\' and label[i+1] == 'x':
+ labelbytes.extend(bytes.fromhex(label[i+2:i+4]))
+ i += 4
+ continue
+ labelbytes.extend(str.encode(label[i]))
+ i = i+1
+ continue
+ label = labelbytes.decode('utf-8')
serial = printable_serial(obj)
return label + ":" + serial
+def write_cert_ext_to_file(f, oid, value, public_key):
+ f.write("[p11-kit-object-v1]\n")
+ f.write("label: ");
+ f.write(tobj['CKA_LABEL'])
+ f.write("\n")
+ f.write("class: x-certificate-extension\n");
+ f.write("object-id: " + oid + "\n")
+ f.write("value: \"" + value + "\"\n")
+ f.write("modifiable: false\n");
+ f.write(public_key)
+
trust_types = {
"CKA_TRUST_DIGITAL_SIGNATURE": "digital-signature",
"CKA_TRUST_NON_REPUDIATION": "non-repudiation",
@@ -151,34 +177,39 @@ openssl_trust = {
"CKA_TRUST_EMAIL_PROTECTION": "emailProtection",
}
+cert_distrust_types = {
+ "CKA_NSS_SERVER_DISTRUST_AFTER": "nss-server-distrust-after",
+ "CKA_NSS_EMAIL_DISTRUST_AFTER": "nss-email-distrust-after",
+}
+
for tobj in objects:
if tobj['CKA_CLASS'] == 'CKO_NSS_TRUST':
key = tobj['CKA_LABEL'] + printable_serial(tobj)
- print "producing trust for " + key
+ print("producing trust for " + key)
trustbits = []
distrustbits = []
openssl_trustflags = []
openssl_distrustflags = []
legacy_trustbits = []
legacy_openssl_trustflags = []
- for t in trust_types.keys():
- if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
+ for t in list(trust_types.keys()):
+ if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
trustbits.append(t)
if t in openssl_trust:
openssl_trustflags.append(openssl_trust[t])
- if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
+ if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
distrustbits.append(t)
if t in openssl_trust:
openssl_distrustflags.append(openssl_trust[t])
- for t in legacy_trust_types.keys():
- if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
+ for t in list(legacy_trust_types.keys()):
+ if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
real_t = legacy_to_real_trust_types[t]
legacy_trustbits.append(real_t)
if real_t in openssl_trust:
legacy_openssl_trustflags.append(openssl_trust[real_t])
- if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
- raise NotImplementedError, 'legacy distrust not supported.\n' + line
+ if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
+ raise NotImplementedError('legacy distrust not supported.\n' + line)
fname = obj_to_filename(tobj)
try:
@@ -186,43 +217,181 @@ for tobj in objects:
except:
obj = None
- if obj != None:
- fname += ".crt"
- else:
- fname += ".p11-kit"
+ # optional debug code, that dumps the parsed input to files
+ #fulldump = "dump-" + fname
+ #dumpf = open(fulldump, 'w')
+ #dumpf.write(str(obj));
+ #dumpf.write(str(tobj));
+ #dumpf.close();
is_legacy = 0
- if tobj.has_key('LEGACY_CKA_TRUST_SERVER_AUTH') or tobj.has_key('LEGACY_CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('LEGACY_CKA_TRUST_CODE_SIGNING'):
+ if 'LEGACY_CKA_TRUST_SERVER_AUTH' in tobj or 'LEGACY_CKA_TRUST_EMAIL_PROTECTION' in tobj or 'LEGACY_CKA_TRUST_CODE_SIGNING' in tobj:
is_legacy = 1
if obj == None:
- raise NotImplementedError, 'found legacy trust without certificate.\n' + line
- legacy_fname = "legacy-default/" + fname
+ raise NotImplementedError('found legacy trust without certificate.\n' + line)
+
+ legacy_fname = "legacy-default/" + fname + ".crt"
f = open(legacy_fname, 'w')
f.write("# alias=%s\n"%tobj['CKA_LABEL'])
f.write("# trust=" + " ".join(legacy_trustbits) + "\n")
if legacy_openssl_trustflags:
f.write("# openssl-trust=" + " ".join(legacy_openssl_trustflags) + "\n")
f.write("-----BEGIN CERTIFICATE-----\n")
- f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
+ temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
+ temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
+ f.write("\n".join(temp_wrapped))
f.write("\n-----END CERTIFICATE-----\n")
f.close()
- if tobj.has_key('CKA_TRUST_SERVER_AUTH') or tobj.has_key('CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('CKA_TRUST_CODE_SIGNING'):
- fname = "legacy-disable/" + fname
- else:
- continue
+ if 'CKA_TRUST_SERVER_AUTH' in tobj or 'CKA_TRUST_EMAIL_PROTECTION' in tobj or 'CKA_TRUST_CODE_SIGNING' in tobj:
+ legacy_fname = "legacy-disable/" + fname + ".crt"
+ f = open(legacy_fname, 'w')
+ f.write("# alias=%s\n"%tobj['CKA_LABEL'])
+ f.write("# trust=" + " ".join(trustbits) + "\n")
+ if openssl_trustflags:
+ f.write("# openssl-trust=" + " ".join(openssl_trustflags) + "\n")
+ f.write("-----BEGIN CERTIFICATE-----\n")
+ f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
+ f.write("\n-----END CERTIFICATE-----\n")
+ f.close()
+
+ # don't produce p11-kit output for legacy certificates
+ continue
+
+ pk = ''
+ cert_comment = ''
+ if obj != None:
+ # must extract the public key from the cert, let's use openssl
+ cert_fname = "cert-" + fname
+ fc = open(cert_fname, 'w')
+ fc.write("-----BEGIN CERTIFICATE-----\n")
+ temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
+ temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
+ fc.write("\n".join(temp_wrapped))
+ fc.write("\n-----END CERTIFICATE-----\n")
+ fc.close();
+ pk_fname = "pubkey-" + fname
+ fpkout = open(pk_fname, "w")
+ dump_pk_command = ["openssl", "x509", "-in", cert_fname, "-noout", "-pubkey"]
+ subprocess.call(dump_pk_command, stdout=fpkout)
+ fpkout.close()
+ with open (pk_fname, "r") as myfile:
+ pk=myfile.read()
+ # obtain certificate information suitable as a comment
+ comment_fname = "comment-" + fname
+ fcout = open(comment_fname, "w")
+ comment_command = ["openssl", "x509", "-in", cert_fname, "-noout", "-text"]
+ subprocess.call(comment_command, stdout=fcout)
+ fcout.close()
+ sed_command = ["sed", "--in-place", "s/^/#/", comment_fname]
+ subprocess.call(sed_command)
+ with open (comment_fname, "r", errors = 'replace') as myfile:
+ cert_comment=myfile.read()
+
+ fname += ".tmp-p11-kit"
f = open(fname, 'w')
+
if obj != None:
- f.write("# alias=%s\n"%tobj['CKA_LABEL'])
- f.write("# trust=" + " ".join(trustbits) + "\n")
- f.write("# distrust=" + " ".join(distrustbits) + "\n")
- if openssl_trustflags:
- f.write("# openssl-trust=" + " ".join(openssl_trustflags) + "\n")
- if openssl_distrustflags:
- f.write("# openssl-distrust=" + " ".join(openssl_distrustflags) + "\n")
+ is_distrusted = False
+ has_server_trust = False
+ has_email_trust = False
+ has_code_trust = False
+
+ if 'CKA_TRUST_SERVER_AUTH' in tobj:
+ if tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED':
+ is_distrusted = True
+ elif tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_TRUSTED_DELEGATOR':
+ has_server_trust = True
+
+ if 'CKA_TRUST_EMAIL_PROTECTION' in tobj:
+ if tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED':
+ is_distrusted = True
+ elif tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_TRUSTED_DELEGATOR':
+ has_email_trust = True
+
+ if 'CKA_TRUST_CODE_SIGNING' in tobj:
+ if tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED':
+ is_distrusted = True
+ elif tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_TRUSTED_DELEGATOR':
+ has_code_trust = True
+
+ if is_distrusted:
+ trust_ext_oid = "1.3.6.1.4.1.3319.6.10.1"
+ trust_ext_value = "0.%06%0a%2b%06%01%04%01%99w%06%0a%01%04 0%1e%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
+ write_cert_ext_to_file(f, trust_ext_oid, trust_ext_value, pk)
+
+ trust_ext_oid = "2.5.29.37"
+ if has_server_trust:
+ if has_email_trust:
+ if has_code_trust:
+ # server + email + code
+ trust_ext_value = "0%2a%06%03U%1d%25%01%01%ff%04 0%1e%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
+ else:
+ # server + email
+ trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01"
+ else:
+ if has_code_trust:
+ # server + code
+ trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
+ else:
+ # server
+ trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%01"
+ else:
+ if has_email_trust:
+ if has_code_trust:
+ # email + code
+ trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%03"
+ else:
+ # email
+ trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%04"
+ else:
+ if has_code_trust:
+ # code
+ trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%03"
+ else:
+ # none
+ trust_ext_value = "0%18%06%03U%1d%25%01%01%ff%04%0e0%0c%06%0a%2b%06%01%04%01%99w%06%0a%10"
+
+ # no 2.5.29.37 for neutral certificates
+ if (is_distrusted or has_server_trust or has_email_trust or has_code_trust):
+ write_cert_ext_to_file(f, trust_ext_oid, trust_ext_value, pk)
+
+ pk = ''
+ f.write("\n")
+
+ f.write("[p11-kit-object-v1]\n")
+ f.write("label: ");
+ f.write(tobj['CKA_LABEL'])
+ f.write("\n")
+ if is_distrusted:
+ f.write("x-distrusted: true\n")
+ elif has_server_trust or has_email_trust or has_code_trust:
+ f.write("trusted: true\n")
+ else:
+ f.write("trusted: false\n")
+
+ # requires p11-kit >= 0.23.4
+ f.write("nss-mozilla-ca-policy: true\n")
+ f.write("modifiable: false\n");
+
+ # requires p11-kit >= 0.23.19
+ for t in list(cert_distrust_types.keys()):
+ if t in obj:
+ value = obj[t]
+ if value == 'CK_FALSE':
+ value = bytearray(1)
+ f.write(cert_distrust_types[t] + ": \"")
+ f.write(urllib.parse.quote(value));
+ f.write("\"\n")
+
f.write("-----BEGIN CERTIFICATE-----\n")
- f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
+ temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
+ temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
+ f.write("\n".join(temp_wrapped))
f.write("\n-----END CERTIFICATE-----\n")
+ f.write(cert_comment)
+ f.write("\n")
+
else:
f.write("[p11-kit-object-v1]\n")
f.write("label: ");
@@ -230,14 +399,15 @@ for tobj in objects:
f.write("\n")
f.write("class: certificate\n")
f.write("certificate-type: x-509\n")
+ f.write("modifiable: false\n");
f.write("issuer: \"");
- f.write(urllib.quote(tobj['CKA_ISSUER']));
+ f.write(urllib.parse.quote(tobj['CKA_ISSUER']));
f.write("\"\n")
f.write("serial-number: \"");
- f.write(urllib.quote(tobj['CKA_SERIAL_NUMBER']));
+ f.write(urllib.parse.quote(tobj['CKA_SERIAL_NUMBER']));
f.write("\"\n")
if (tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED'):
f.write("x-distrusted: true\n")
f.write("\n\n")
f.close()
- print " -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags)
+ print(" -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags))
@@ -1,4 +1,7 @@
+#etc/pki
+#etc/pki/ca-trust
+#etc/pki/ca-trust/source
+etc/pki/ca-trust/source/ca-bundle.trust.p11-kit
etc/ssl/cert.pem
-#etc/ssl/certs
etc/ssl/certs/ca-bundle.crt
etc/ssl/certs/ca-bundle.trust.crt
@@ -24,7 +24,7 @@
include Config
-VER = 20210611
+VER = 20210819
THISAPP = ca-certificates
DIR_APP = $(DIR_SRC)/$(THISAPP)