ca-certificates: Update to work with python3 version of certdata2pem.py

- Implement python3 version of certdata2pem.py script from fedora
- Modify build.sh to work with python3 script that uses p11-kit based on fedora
   approach - https://src.fedoraproject.org/rpms/ca-certificates/tree/rawhide
- Extraction of cert files now uses p11-kit which requires libtasn1 as a build
   dependency
- Updated rootfile
- Updated ca-certificates installed into a vm and confirmed to download a file from an
   https site with the same results as with existing ca-certfictaes system

Tested-by: Adolf Belka <adolf.belka@ipfire.org>
Signed-off-by: Adolf Belka <adolf.belka@ipfire.org>
Reviewed-by: Michael Tremer <michael.tremer@ipfire.org>
Signed-off-by: Arne Fitzenreiter <arne_f@ipfire.org>
This commit is contained in:
Adolf Belka
2021-08-20 22:04:25 +02:00
committed by Arne Fitzenreiter
parent 5b4fc4c849
commit 043abb984b
4 changed files with 249 additions and 68 deletions

View File

@@ -3,13 +3,34 @@
set -e
# Create file layout.
mkdir -pv certs certs/legacy-default certs/legacy-disable
mkdir -pv certs
mkdir -pv /etc/pki/ca-trust/source
cp certdata.txt certs
cd certs
python ../certdata2pem.py
python3 ../certdata2pem.py
cd ..
cat <<EOF > ca-bundle.trust.p11-kit
# This is a bundle of X.509 certificates of public Certificate
# Authorities. It was generated from the Mozilla root CA list.
# These certificates and trust/distrust attributes use the file format accepted
# by the p11-kit-trust module.
#
# Source: mozilla/security/nss/lib/ckfw/builtins/certdata.txt
#
EOF
P11FILES=`find certs -name \*.tmp-p11-kit | wc -l`
if [ $P11FILES -ne 0 ]; then
for p in certs/*.tmp-p11-kit; do
cat "$p" >> /etc/pki/ca-trust/source/ca-bundle.trust.p11-kit
done
fi
cat <<EOF > ca-bundle.crt
# This is a bundle of X.509 certificates of public Certificate
# Authorities. It was generated from the Mozilla root CA list.
@@ -28,24 +49,11 @@ cat <<EOF > ca-bundle.trust.crt
#
EOF
for f in certs/*.crt; do
[ -z "${f}" ] && continue
trust extract --comment --filter=certificates --format=openssl-bundle --overwrite ca-bundle.trust
cat ca-bundle.trust >> ca-bundle.trust.crt
tbits=$(sed -n '/^# openssl-trust/{s/^.*=//;p;}' ${f})
case "${tbits}" in
*serverAuth*)
openssl x509 -text -in "${f}" >> ca-bundle.crt
;;
esac
trust extract --comment --filter=ca-anchors --format=pem-bundle --overwrite --purpose server-auth ca-bundle
cat ca-bundle >> ca-bundle.crt
if [ -n "$tbits" ]; then
targs=""
for t in ${tbits}; do
targs="${targs} -addtrust ${t}"
done
openssl x509 -text -in "${f}" -trustout $targs >> ca-bundle.trust.crt
fi
done
exit 0
exit 0

View File

@@ -26,16 +26,17 @@ import os.path
import re
import sys
import textwrap
import urllib
import urllib.request, urllib.parse, urllib.error
import subprocess
objects = []
def printable_serial(obj):
return ".".join(map(lambda x:str(ord(x)), obj['CKA_SERIAL_NUMBER']))
return ".".join([str(x) for x in obj['CKA_SERIAL_NUMBER']])
# Dirty file parser.
in_data, in_multiline, in_obj = False, False, False
field, type, value, obj = None, None, None, dict()
field, ftype, value, binval, obj = None, None, None, bytearray(), dict()
for line in open('certdata.txt', 'r'):
# Ignore the file header.
if not in_data:
@@ -55,33 +56,36 @@ for line in open('certdata.txt', 'r'):
continue
if in_multiline:
if not line.startswith('END'):
if type == 'MULTILINE_OCTAL':
if ftype == 'MULTILINE_OCTAL':
line = line.strip()
for i in re.finditer(r'\\([0-3][0-7][0-7])', line):
value += chr(int(i.group(1), 8))
integ = int(i.group(1), 8)
binval.extend((integ).to_bytes(1, sys.byteorder))
obj[field] = binval
else:
value += line
obj[field] = value
continue
obj[field] = value
in_multiline = False
continue
if line.startswith('CKA_CLASS'):
in_obj = True
line_parts = line.strip().split(' ', 2)
if len(line_parts) > 2:
field, type = line_parts[0:2]
field, ftype = line_parts[0:2]
value = ' '.join(line_parts[2:])
elif len(line_parts) == 2:
field, type = line_parts
field, ftype = line_parts
value = None
else:
raise NotImplementedError, 'line_parts < 2 not supported.\n' + line
if type == 'MULTILINE_OCTAL':
raise NotImplementedError('line_parts < 2 not supported.\n' + line)
if ftype == 'MULTILINE_OCTAL':
in_multiline = True
value = ""
binval = bytearray()
continue
obj[field] = value
if len(obj.items()) > 0:
if len(list(obj.items())) > 0:
objects.append(obj)
# Build up trust database.
@@ -91,7 +95,7 @@ for obj in objects:
continue
key = obj['CKA_LABEL'] + printable_serial(obj)
trustmap[key] = obj
print " added trust", key
print(" added trust", key)
# Build up cert database.
certmap = dict()
@@ -100,7 +104,7 @@ for obj in objects:
continue
key = obj['CKA_LABEL'] + printable_serial(obj)
certmap[key] = obj
print " added cert", key
print(" added cert", key)
def obj_to_filename(obj):
label = obj['CKA_LABEL'][1:-1]
@@ -109,10 +113,32 @@ def obj_to_filename(obj):
.replace('(', '=')\
.replace(')', '=')\
.replace(',', '_')
label = re.sub(r'\\x[0-9a-fA-F]{2}', lambda m:chr(int(m.group(0)[2:], 16)), label)
labelbytes = bytearray()
i = 0
imax = len(label)
while i < imax:
if i < imax-3 and label[i] == '\\' and label[i+1] == 'x':
labelbytes.extend(bytes.fromhex(label[i+2:i+4]))
i += 4
continue
labelbytes.extend(str.encode(label[i]))
i = i+1
continue
label = labelbytes.decode('utf-8')
serial = printable_serial(obj)
return label + ":" + serial
def write_cert_ext_to_file(f, oid, value, public_key):
f.write("[p11-kit-object-v1]\n")
f.write("label: ");
f.write(tobj['CKA_LABEL'])
f.write("\n")
f.write("class: x-certificate-extension\n");
f.write("object-id: " + oid + "\n")
f.write("value: \"" + value + "\"\n")
f.write("modifiable: false\n");
f.write(public_key)
trust_types = {
"CKA_TRUST_DIGITAL_SIGNATURE": "digital-signature",
"CKA_TRUST_NON_REPUDIATION": "non-repudiation",
@@ -151,34 +177,39 @@ openssl_trust = {
"CKA_TRUST_EMAIL_PROTECTION": "emailProtection",
}
cert_distrust_types = {
"CKA_NSS_SERVER_DISTRUST_AFTER": "nss-server-distrust-after",
"CKA_NSS_EMAIL_DISTRUST_AFTER": "nss-email-distrust-after",
}
for tobj in objects:
if tobj['CKA_CLASS'] == 'CKO_NSS_TRUST':
key = tobj['CKA_LABEL'] + printable_serial(tobj)
print "producing trust for " + key
print("producing trust for " + key)
trustbits = []
distrustbits = []
openssl_trustflags = []
openssl_distrustflags = []
legacy_trustbits = []
legacy_openssl_trustflags = []
for t in trust_types.keys():
if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
for t in list(trust_types.keys()):
if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
trustbits.append(t)
if t in openssl_trust:
openssl_trustflags.append(openssl_trust[t])
if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
distrustbits.append(t)
if t in openssl_trust:
openssl_distrustflags.append(openssl_trust[t])
for t in legacy_trust_types.keys():
if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
for t in list(legacy_trust_types.keys()):
if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
real_t = legacy_to_real_trust_types[t]
legacy_trustbits.append(real_t)
if real_t in openssl_trust:
legacy_openssl_trustflags.append(openssl_trust[real_t])
if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
raise NotImplementedError, 'legacy distrust not supported.\n' + line
if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
raise NotImplementedError('legacy distrust not supported.\n' + line)
fname = obj_to_filename(tobj)
try:
@@ -186,43 +217,181 @@ for tobj in objects:
except:
obj = None
if obj != None:
fname += ".crt"
else:
fname += ".p11-kit"
# optional debug code, that dumps the parsed input to files
#fulldump = "dump-" + fname
#dumpf = open(fulldump, 'w')
#dumpf.write(str(obj));
#dumpf.write(str(tobj));
#dumpf.close();
is_legacy = 0
if tobj.has_key('LEGACY_CKA_TRUST_SERVER_AUTH') or tobj.has_key('LEGACY_CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('LEGACY_CKA_TRUST_CODE_SIGNING'):
if 'LEGACY_CKA_TRUST_SERVER_AUTH' in tobj or 'LEGACY_CKA_TRUST_EMAIL_PROTECTION' in tobj or 'LEGACY_CKA_TRUST_CODE_SIGNING' in tobj:
is_legacy = 1
if obj == None:
raise NotImplementedError, 'found legacy trust without certificate.\n' + line
legacy_fname = "legacy-default/" + fname
raise NotImplementedError('found legacy trust without certificate.\n' + line)
legacy_fname = "legacy-default/" + fname + ".crt"
f = open(legacy_fname, 'w')
f.write("# alias=%s\n"%tobj['CKA_LABEL'])
f.write("# trust=" + " ".join(legacy_trustbits) + "\n")
if legacy_openssl_trustflags:
f.write("# openssl-trust=" + " ".join(legacy_openssl_trustflags) + "\n")
f.write("-----BEGIN CERTIFICATE-----\n")
f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
f.write("\n".join(temp_wrapped))
f.write("\n-----END CERTIFICATE-----\n")
f.close()
if tobj.has_key('CKA_TRUST_SERVER_AUTH') or tobj.has_key('CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('CKA_TRUST_CODE_SIGNING'):
fname = "legacy-disable/" + fname
else:
continue
f = open(fname, 'w')
if 'CKA_TRUST_SERVER_AUTH' in tobj or 'CKA_TRUST_EMAIL_PROTECTION' in tobj or 'CKA_TRUST_CODE_SIGNING' in tobj:
legacy_fname = "legacy-disable/" + fname + ".crt"
f = open(legacy_fname, 'w')
f.write("# alias=%s\n"%tobj['CKA_LABEL'])
f.write("# trust=" + " ".join(trustbits) + "\n")
if openssl_trustflags:
f.write("# openssl-trust=" + " ".join(openssl_trustflags) + "\n")
f.write("-----BEGIN CERTIFICATE-----\n")
f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
f.write("\n-----END CERTIFICATE-----\n")
f.close()
# don't produce p11-kit output for legacy certificates
continue
pk = ''
cert_comment = ''
if obj != None:
f.write("# alias=%s\n"%tobj['CKA_LABEL'])
f.write("# trust=" + " ".join(trustbits) + "\n")
f.write("# distrust=" + " ".join(distrustbits) + "\n")
if openssl_trustflags:
f.write("# openssl-trust=" + " ".join(openssl_trustflags) + "\n")
if openssl_distrustflags:
f.write("# openssl-distrust=" + " ".join(openssl_distrustflags) + "\n")
# must extract the public key from the cert, let's use openssl
cert_fname = "cert-" + fname
fc = open(cert_fname, 'w')
fc.write("-----BEGIN CERTIFICATE-----\n")
temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
fc.write("\n".join(temp_wrapped))
fc.write("\n-----END CERTIFICATE-----\n")
fc.close();
pk_fname = "pubkey-" + fname
fpkout = open(pk_fname, "w")
dump_pk_command = ["openssl", "x509", "-in", cert_fname, "-noout", "-pubkey"]
subprocess.call(dump_pk_command, stdout=fpkout)
fpkout.close()
with open (pk_fname, "r") as myfile:
pk=myfile.read()
# obtain certificate information suitable as a comment
comment_fname = "comment-" + fname
fcout = open(comment_fname, "w")
comment_command = ["openssl", "x509", "-in", cert_fname, "-noout", "-text"]
subprocess.call(comment_command, stdout=fcout)
fcout.close()
sed_command = ["sed", "--in-place", "s/^/#/", comment_fname]
subprocess.call(sed_command)
with open (comment_fname, "r", errors = 'replace') as myfile:
cert_comment=myfile.read()
fname += ".tmp-p11-kit"
f = open(fname, 'w')
if obj != None:
is_distrusted = False
has_server_trust = False
has_email_trust = False
has_code_trust = False
if 'CKA_TRUST_SERVER_AUTH' in tobj:
if tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED':
is_distrusted = True
elif tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_TRUSTED_DELEGATOR':
has_server_trust = True
if 'CKA_TRUST_EMAIL_PROTECTION' in tobj:
if tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED':
is_distrusted = True
elif tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_TRUSTED_DELEGATOR':
has_email_trust = True
if 'CKA_TRUST_CODE_SIGNING' in tobj:
if tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED':
is_distrusted = True
elif tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_TRUSTED_DELEGATOR':
has_code_trust = True
if is_distrusted:
trust_ext_oid = "1.3.6.1.4.1.3319.6.10.1"
trust_ext_value = "0.%06%0a%2b%06%01%04%01%99w%06%0a%01%04 0%1e%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
write_cert_ext_to_file(f, trust_ext_oid, trust_ext_value, pk)
trust_ext_oid = "2.5.29.37"
if has_server_trust:
if has_email_trust:
if has_code_trust:
# server + email + code
trust_ext_value = "0%2a%06%03U%1d%25%01%01%ff%04 0%1e%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
else:
# server + email
trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01"
else:
if has_code_trust:
# server + code
trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
else:
# server
trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%01"
else:
if has_email_trust:
if has_code_trust:
# email + code
trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%03"
else:
# email
trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%04"
else:
if has_code_trust:
# code
trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%03"
else:
# none
trust_ext_value = "0%18%06%03U%1d%25%01%01%ff%04%0e0%0c%06%0a%2b%06%01%04%01%99w%06%0a%10"
# no 2.5.29.37 for neutral certificates
if (is_distrusted or has_server_trust or has_email_trust or has_code_trust):
write_cert_ext_to_file(f, trust_ext_oid, trust_ext_value, pk)
pk = ''
f.write("\n")
f.write("[p11-kit-object-v1]\n")
f.write("label: ");
f.write(tobj['CKA_LABEL'])
f.write("\n")
if is_distrusted:
f.write("x-distrusted: true\n")
elif has_server_trust or has_email_trust or has_code_trust:
f.write("trusted: true\n")
else:
f.write("trusted: false\n")
# requires p11-kit >= 0.23.4
f.write("nss-mozilla-ca-policy: true\n")
f.write("modifiable: false\n");
# requires p11-kit >= 0.23.19
for t in list(cert_distrust_types.keys()):
if t in obj:
value = obj[t]
if value == 'CK_FALSE':
value = bytearray(1)
f.write(cert_distrust_types[t] + ": \"")
f.write(urllib.parse.quote(value));
f.write("\"\n")
f.write("-----BEGIN CERTIFICATE-----\n")
f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
f.write("\n".join(temp_wrapped))
f.write("\n-----END CERTIFICATE-----\n")
f.write(cert_comment)
f.write("\n")
else:
f.write("[p11-kit-object-v1]\n")
f.write("label: ");
@@ -230,14 +399,15 @@ for tobj in objects:
f.write("\n")
f.write("class: certificate\n")
f.write("certificate-type: x-509\n")
f.write("modifiable: false\n");
f.write("issuer: \"");
f.write(urllib.quote(tobj['CKA_ISSUER']));
f.write(urllib.parse.quote(tobj['CKA_ISSUER']));
f.write("\"\n")
f.write("serial-number: \"");
f.write(urllib.quote(tobj['CKA_SERIAL_NUMBER']));
f.write(urllib.parse.quote(tobj['CKA_SERIAL_NUMBER']));
f.write("\"\n")
if (tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED'):
f.write("x-distrusted: true\n")
f.write("\n\n")
f.close()
print " -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags)
print(" -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags))

View File

@@ -1,4 +1,7 @@
#etc/pki
#etc/pki/ca-trust
#etc/pki/ca-trust/source
etc/pki/ca-trust/source/ca-bundle.trust.p11-kit
etc/ssl/cert.pem
#etc/ssl/certs
etc/ssl/certs/ca-bundle.crt
etc/ssl/certs/ca-bundle.trust.crt

View File

@@ -24,7 +24,7 @@
include Config
VER = 20210611
VER = 20210819
THISAPP = ca-certificates
DIR_APP = $(DIR_SRC)/$(THISAPP)