diff --git a/configure.def b/configure.def index 2d9efce0b1..068969cc16 100644 --- a/configure.def +++ b/configure.def @@ -363,7 +363,7 @@ for _dir in external/libblake2 external/libpicosat \ external/liblinenoise external/libsqlite external/libfetch \ external compat libpkg libpkg/repo libpkg/repo/binary src \ external/libucl external/libelf tests docs tests/fuzz \ - external/liblua external/libyxml scripts external/libder \ + external/liblua scripts external/libder \ external/libecc; do make_template "$_dir/Makefile.in" define_append CONF_GEN_FILES "$_dir/Makefile.in" diff --git a/docs/pkg-install.8 b/docs/pkg-install.8 index d92a81967b..af4aabdd98 100644 --- a/docs/pkg-install.8 +++ b/docs/pkg-install.8 @@ -97,7 +97,7 @@ setting in .Pa pkg.conf . .Pp If the vulnerability database -.Pa ( vuln.xml ) +.Pa ( freebsd-osv.json ) is available .Pq see Xr pkg-audit 8 , the summary will flag any packages with known vulnerabilities and diff --git a/docs/pkg-upgrade.8 b/docs/pkg-upgrade.8 index daab2257c5..6bcbe52b14 100644 --- a/docs/pkg-upgrade.8 +++ b/docs/pkg-upgrade.8 @@ -101,7 +101,7 @@ The option refuses those removals altogether. .Pp If the vulnerability database -.Pa ( vuln.xml ) +.Pa ( freebsd-osv.json ) is available .Pq see Xr pkg-audit 8 , the summary will flag any packages with known vulnerabilities and diff --git a/docs/pkg.conf.5 b/docs/pkg.conf.5 index 9ca63066fd..148a4262f9 100644 --- a/docs/pkg.conf.5 +++ b/docs/pkg.conf.5 @@ -456,10 +456,16 @@ configuration file. Default: pkg+http, pkg+https, https, http, file, ssh, tcp. .It Cm VULNXML_SITE: string Specifies the URL to fetch the -.Pa vuln.xml +.Pa freebsd-osv.json vulnerability database from. Default: -.Pa https://vuxml.freebsd.org/freebsd/vuln.xml.xz . +.Pa https://raw.githubusercontent.com/illuusio/freebsd-osv/refs/heads/main/db/freebsd-osv.json . +.It Cm VULNXML_SITE: string +Specifies the URL to fetch the +.Pa freebsd-osv.json +vulnerability database from. +Default: +.Pa https://raw.githubusercontent.com/illuusio/freebsd-osv/refs/heads/main/db/freebsd-osv.json . .It Cm WARN_SIZE_LIMIT: integer When download and total change is less than than this option, don't ask the user to confirm the desired action. diff --git a/external/Makefile.in b/external/Makefile.in index ef6db8726f..133d9e64aa 100644 --- a/external/Makefile.in +++ b/external/Makefile.in @@ -1,5 +1,5 @@ include @builddir@/mk/defs.mk -DIRS= libblake2 libpicosat liblinenoise libfetch libsqlite libucl liblua libyxml libder libecc +DIRS= libblake2 libpicosat liblinenoise libfetch libsqlite libucl liblua libder libecc @if libelf-internal DIRS+= libelf @endif diff --git a/external/libyxml/Makefile.in b/external/libyxml/Makefile.in deleted file mode 100644 index e773555825..0000000000 --- a/external/libyxml/Makefile.in +++ /dev/null @@ -1,13 +0,0 @@ -include @builddir@/mk/defs.mk - -LIB= yxml - -SRCS= yxml.c - -LOCAL_CFLAGS= -Wno-unused-parameter -I. -I$(top_srcdir)/external/yxml - -VPATH= $(top_srcdir)/external/yxml - -include $(MK)/static-lib.mk - - diff --git a/external/yxml/yxml.c b/external/yxml/yxml.c deleted file mode 100644 index 23d04a9725..0000000000 --- a/external/yxml/yxml.c +++ /dev/null @@ -1,1060 +0,0 @@ -/* This file is generated by yxml-gen.pl using yxml-states and yxml.c.in as input files. - * It is preferable to edit those files instead of this one if you want to make a change. - * The source files can be found through the homepage: https://dev.yorhel.nl/yxml */ - -/* Copyright (c) 2013-2014 Yoran Heling - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#include -#include - -typedef enum { - YXMLS_string, - YXMLS_attr0, - YXMLS_attr1, - YXMLS_attr2, - YXMLS_attr3, - YXMLS_attr4, - YXMLS_cd0, - YXMLS_cd1, - YXMLS_cd2, - YXMLS_comment0, - YXMLS_comment1, - YXMLS_comment2, - YXMLS_comment3, - YXMLS_comment4, - YXMLS_dt0, - YXMLS_dt1, - YXMLS_dt2, - YXMLS_dt3, - YXMLS_dt4, - YXMLS_elem0, - YXMLS_elem1, - YXMLS_elem2, - YXMLS_elem3, - YXMLS_enc0, - YXMLS_enc1, - YXMLS_enc2, - YXMLS_enc3, - YXMLS_etag0, - YXMLS_etag1, - YXMLS_etag2, - YXMLS_init, - YXMLS_le0, - YXMLS_le1, - YXMLS_le2, - YXMLS_le3, - YXMLS_lee1, - YXMLS_lee2, - YXMLS_leq0, - YXMLS_misc0, - YXMLS_misc1, - YXMLS_misc2, - YXMLS_misc2a, - YXMLS_misc3, - YXMLS_pi0, - YXMLS_pi1, - YXMLS_pi2, - YXMLS_pi3, - YXMLS_pi4, - YXMLS_std0, - YXMLS_std1, - YXMLS_std2, - YXMLS_std3, - YXMLS_ver0, - YXMLS_ver1, - YXMLS_ver2, - YXMLS_ver3, - YXMLS_xmldecl0, - YXMLS_xmldecl1, - YXMLS_xmldecl2, - YXMLS_xmldecl3, - YXMLS_xmldecl4, - YXMLS_xmldecl5, - YXMLS_xmldecl6, - YXMLS_xmldecl7, - YXMLS_xmldecl8, - YXMLS_xmldecl9 -} yxml_state_t; - - -#define yxml_isChar(c) 1 -/* 0xd should be part of SP, too, but yxml_parse() already normalizes that into 0xa */ -#define yxml_isSP(c) (c == 0x20 || c == 0x09 || c == 0x0a) -#define yxml_isAlpha(c) ((c|32)-'a' < 26) -#define yxml_isNum(c) (c-'0' < 10) -#define yxml_isHex(c) (yxml_isNum(c) || (c|32)-'a' < 6) -#define yxml_isEncName(c) (yxml_isAlpha(c) || yxml_isNum(c) || c == '.' || c == '_' || c == '-') -#define yxml_isNameStart(c) (yxml_isAlpha(c) || c == ':' || c == '_' || c >= 128) -#define yxml_isName(c) (yxml_isNameStart(c) || yxml_isNum(c) || c == '-' || c == '.') -/* XXX: The valid characters are dependent on the quote char, hence the access to x->quote */ -#define yxml_isAttValue(c) (yxml_isChar(c) && c != x->quote && c != '<' && c != '&') -/* Anything between '&' and ';', the yxml_ref* functions will do further - * validation. Strictly speaking, this is "yxml_isName(c) || c == '#'", but - * this parser doesn't understand entities with '.', ':', etc, anwyay. */ -#define yxml_isRef(c) (yxml_isNum(c) || yxml_isAlpha(c) || c == '#') - -#define INTFROM5CHARS(a, b, c, d, e) ((((uint64_t)(a))<<32) | (((uint64_t)(b))<<24) | (((uint64_t)(c))<<16) | (((uint64_t)(d))<<8) | (uint64_t)(e)) - - -/* Set the given char value to ch (0<=ch<=255). */ -static inline void yxml_setchar(char *dest, unsigned ch) { - *(unsigned char *)dest = ch; -} - - -/* Similar to yxml_setchar(), but will convert ch (any valid unicode point) to - * UTF-8 and appends a '\0'. dest must have room for at least 5 bytes. */ -static void yxml_setutf8(char *dest, unsigned ch) { - if(ch <= 0x007F) - yxml_setchar(dest++, ch); - else if(ch <= 0x07FF) { - yxml_setchar(dest++, 0xC0 | (ch>>6)); - yxml_setchar(dest++, 0x80 | (ch & 0x3F)); - } else if(ch <= 0xFFFF) { - yxml_setchar(dest++, 0xE0 | (ch>>12)); - yxml_setchar(dest++, 0x80 | ((ch>>6) & 0x3F)); - yxml_setchar(dest++, 0x80 | (ch & 0x3F)); - } else { - yxml_setchar(dest++, 0xF0 | (ch>>18)); - yxml_setchar(dest++, 0x80 | ((ch>>12) & 0x3F)); - yxml_setchar(dest++, 0x80 | ((ch>>6) & 0x3F)); - yxml_setchar(dest++, 0x80 | (ch & 0x3F)); - } - *dest = 0; -} - - -static inline yxml_ret_t yxml_datacontent(yxml_t *x, unsigned ch) { - yxml_setchar(x->data, ch); - x->data[1] = 0; - return YXML_CONTENT; -} - - -static inline yxml_ret_t yxml_datapi1(yxml_t *x, unsigned ch) { - yxml_setchar(x->data, ch); - x->data[1] = 0; - return YXML_PICONTENT; -} - - -static inline yxml_ret_t yxml_datapi2(yxml_t *x, unsigned ch) { - x->data[0] = '?'; - yxml_setchar(x->data+1, ch); - x->data[2] = 0; - return YXML_PICONTENT; -} - - -static inline yxml_ret_t yxml_datacd1(yxml_t *x, unsigned ch) { - x->data[0] = ']'; - yxml_setchar(x->data+1, ch); - x->data[2] = 0; - return YXML_CONTENT; -} - - -static inline yxml_ret_t yxml_datacd2(yxml_t *x, unsigned ch) { - x->data[0] = ']'; - x->data[1] = ']'; - yxml_setchar(x->data+2, ch); - x->data[3] = 0; - return YXML_CONTENT; -} - - -static inline yxml_ret_t yxml_dataattr(yxml_t *x, unsigned ch) { - /* Normalize attribute values according to the XML spec section 3.3.3. */ - yxml_setchar(x->data, ch == 0x9 || ch == 0xa ? 0x20 : ch); - x->data[1] = 0; - return YXML_ATTRVAL; -} - - -static yxml_ret_t yxml_pushstack(yxml_t *x, char **res, unsigned ch) { - if(x->stacklen+2 >= x->stacksize) - return YXML_ESTACK; - x->stacklen++; - *res = (char *)x->stack+x->stacklen; - x->stack[x->stacklen] = ch; - x->stacklen++; - x->stack[x->stacklen] = 0; - return YXML_OK; -} - - -static yxml_ret_t yxml_pushstackc(yxml_t *x, unsigned ch) { - if(x->stacklen+1 >= x->stacksize) - return YXML_ESTACK; - x->stack[x->stacklen] = ch; - x->stacklen++; - x->stack[x->stacklen] = 0; - return YXML_OK; -} - - -static void yxml_popstack(yxml_t *x) { - do - x->stacklen--; - while(x->stack[x->stacklen]); -} - - -static inline yxml_ret_t yxml_elemstart (yxml_t *x, unsigned ch) { return yxml_pushstack(x, &x->elem, ch); } -static inline yxml_ret_t yxml_elemname (yxml_t *x, unsigned ch) { return yxml_pushstackc(x, ch); } -static inline yxml_ret_t yxml_elemnameend(yxml_t *x, unsigned ch) { return YXML_ELEMSTART; } - - -/* Also used in yxml_elemcloseend(), since this function just removes the last - * element from the stack and returns ELEMEND. */ -static yxml_ret_t yxml_selfclose(yxml_t *x, unsigned ch) { - yxml_popstack(x); - if(x->stacklen) { - x->elem = (char *)x->stack+x->stacklen-1; - while(*(x->elem-1)) - x->elem--; - return YXML_ELEMEND; - } - x->elem = (char *)x->stack; - x->state = YXMLS_misc3; - return YXML_ELEMEND; -} - - -static inline yxml_ret_t yxml_elemclose(yxml_t *x, unsigned ch) { - if(*((unsigned char *)x->elem) != ch) - return YXML_ECLOSE; - x->elem++; - return YXML_OK; -} - - -static inline yxml_ret_t yxml_elemcloseend(yxml_t *x, unsigned ch) { - if(*x->elem) - return YXML_ECLOSE; - return yxml_selfclose(x, ch); -} - - -static inline yxml_ret_t yxml_attrstart (yxml_t *x, unsigned ch) { return yxml_pushstack(x, &x->attr, ch); } -static inline yxml_ret_t yxml_attrname (yxml_t *x, unsigned ch) { return yxml_pushstackc(x, ch); } -static inline yxml_ret_t yxml_attrnameend(yxml_t *x, unsigned ch) { return YXML_ATTRSTART; } -static inline yxml_ret_t yxml_attrvalend (yxml_t *x, unsigned ch) { yxml_popstack(x); return YXML_ATTREND; } - - -static inline yxml_ret_t yxml_pistart (yxml_t *x, unsigned ch) { return yxml_pushstack(x, &x->pi, ch); } -static inline yxml_ret_t yxml_piname (yxml_t *x, unsigned ch) { return yxml_pushstackc(x, ch); } -static inline yxml_ret_t yxml_piabort (yxml_t *x, unsigned ch) { yxml_popstack(x); return YXML_OK; } -static inline yxml_ret_t yxml_pinameend(yxml_t *x, unsigned ch) { - return (x->pi[0]|32) == 'x' && (x->pi[1]|32) == 'm' && (x->pi[2]|32) == 'l' && !x->pi[3] ? YXML_ESYN : YXML_PISTART; -} -static inline yxml_ret_t yxml_pivalend (yxml_t *x, unsigned ch) { yxml_popstack(x); x->pi = (char *)x->stack; return YXML_PIEND; } - - -static inline yxml_ret_t yxml_refstart(yxml_t *x, unsigned ch) { - memset(x->data, 0, sizeof(x->data)); - x->reflen = 0; - return YXML_OK; -} - - -static yxml_ret_t yxml_ref(yxml_t *x, unsigned ch) { - if(x->reflen >= sizeof(x->data)-1) - return YXML_EREF; - yxml_setchar(x->data+x->reflen, ch); - x->reflen++; - return YXML_OK; -} - - -static yxml_ret_t yxml_refend(yxml_t *x, yxml_ret_t ret) { - unsigned char *r = (unsigned char *)x->data; - unsigned ch = 0; - if(*r == '#') { - if(r[1] == 'x') - for(r += 2; yxml_isHex((unsigned)*r); r++) - ch = (ch<<4) + (*r <= '9' ? *r-'0' : (*r|32)-'a' + 10); - else - for(r++; yxml_isNum((unsigned)*r); r++) - ch = (ch*10) + (*r-'0'); - if(*r) - ch = 0; - } else { - uint64_t i = INTFROM5CHARS(r[0], r[1], r[2], r[3], r[4]); - ch = - i == INTFROM5CHARS('l','t', 0, 0, 0) ? '<' : - i == INTFROM5CHARS('g','t', 0, 0, 0) ? '>' : - i == INTFROM5CHARS('a','m','p', 0, 0) ? '&' : - i == INTFROM5CHARS('a','p','o','s',0) ? '\'': - i == INTFROM5CHARS('q','u','o','t',0) ? '"' : 0; - } - - /* Codepoints not allowed in the XML 1.1 definition of a Char */ - if(!ch || ch > 0x10FFFF || ch == 0xFFFE || ch == 0xFFFF || (ch-0xDFFF) < 0x7FF) - return YXML_EREF; - yxml_setutf8(x->data, ch); - return ret; -} - - -static inline yxml_ret_t yxml_refcontent(yxml_t *x, unsigned ch) { return yxml_refend(x, YXML_CONTENT); } -static inline yxml_ret_t yxml_refattrval(yxml_t *x, unsigned ch) { return yxml_refend(x, YXML_ATTRVAL); } - - -void yxml_init(yxml_t *x, void *stack, size_t stacksize) { - memset(x, 0, sizeof(*x)); - x->line = 1; - x->stack = (unsigned char*)stack; - x->stacksize = stacksize; - *x->stack = 0; - x->elem = x->pi = x->attr = (char *)x->stack; - x->state = YXMLS_init; -} - - -yxml_ret_t yxml_parse(yxml_t *x, int _ch) { - /* Ensure that characters are in the range of 0..255 rather than -126..125. - * All character comparisons are done with positive integers. */ - unsigned ch = (unsigned)(_ch+256) & 0xff; - if(!ch) - return YXML_ESYN; - x->total++; - - /* End-of-Line normalization, "\rX", "\r\n" and "\n" are recognized and - * normalized to a single '\n' as per XML 1.0 section 2.11. XML 1.1 adds - * some non-ASCII character sequences to this list, but we can only handle - * ASCII here without making assumptions about the input encoding. */ - if(x->ignore == ch) { - x->ignore = 0; - return YXML_OK; - } - x->ignore = (ch == 0xd) * 0xa; - if(ch == 0xa || ch == 0xd) { - ch = 0xa; - x->line++; - x->byte = 0; - } - x->byte++; - - switch((yxml_state_t)x->state) { - case YXMLS_string: - if(ch == *x->string) { - x->string++; - if(!*x->string) - x->state = x->nextstate; - return YXML_OK; - } - break; - case YXMLS_attr0: - if(yxml_isName(ch)) - return yxml_attrname(x, ch); - if(yxml_isSP(ch)) { - x->state = YXMLS_attr1; - return yxml_attrnameend(x, ch); - } - if(ch == (unsigned char)'=') { - x->state = YXMLS_attr2; - return yxml_attrnameend(x, ch); - } - break; - case YXMLS_attr1: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'=') { - x->state = YXMLS_attr2; - return YXML_OK; - } - break; - case YXMLS_attr2: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'\'' || ch == (unsigned char)'"') { - x->state = YXMLS_attr3; - x->quote = ch; - return YXML_OK; - } - break; - case YXMLS_attr3: - if(yxml_isAttValue(ch)) - return yxml_dataattr(x, ch); - if(ch == (unsigned char)'&') { - x->state = YXMLS_attr4; - return yxml_refstart(x, ch); - } - if(x->quote == ch) { - x->state = YXMLS_elem2; - return yxml_attrvalend(x, ch); - } - break; - case YXMLS_attr4: - if(yxml_isRef(ch)) - return yxml_ref(x, ch); - if(ch == (unsigned char)'\x3b') { - x->state = YXMLS_attr3; - return yxml_refattrval(x, ch); - } - break; - case YXMLS_cd0: - if(ch == (unsigned char)']') { - x->state = YXMLS_cd1; - return YXML_OK; - } - if(yxml_isChar(ch)) - return yxml_datacontent(x, ch); - break; - case YXMLS_cd1: - if(ch == (unsigned char)']') { - x->state = YXMLS_cd2; - return YXML_OK; - } - if(yxml_isChar(ch)) { - x->state = YXMLS_cd0; - return yxml_datacd1(x, ch); - } - break; - case YXMLS_cd2: - if(ch == (unsigned char)']') - return yxml_datacontent(x, ch); - if(ch == (unsigned char)'>') { - x->state = YXMLS_misc2; - return YXML_OK; - } - if(yxml_isChar(ch)) { - x->state = YXMLS_cd0; - return yxml_datacd2(x, ch); - } - break; - case YXMLS_comment0: - if(ch == (unsigned char)'-') { - x->state = YXMLS_comment1; - return YXML_OK; - } - break; - case YXMLS_comment1: - if(ch == (unsigned char)'-') { - x->state = YXMLS_comment2; - return YXML_OK; - } - break; - case YXMLS_comment2: - if(ch == (unsigned char)'-') { - x->state = YXMLS_comment3; - return YXML_OK; - } - if(yxml_isChar(ch)) - return YXML_OK; - break; - case YXMLS_comment3: - if(ch == (unsigned char)'-') { - x->state = YXMLS_comment4; - return YXML_OK; - } - if(yxml_isChar(ch)) { - x->state = YXMLS_comment2; - return YXML_OK; - } - break; - case YXMLS_comment4: - if(ch == (unsigned char)'>') { - x->state = x->nextstate; - return YXML_OK; - } - break; - case YXMLS_dt0: - if(ch == (unsigned char)'>') { - x->state = YXMLS_misc1; - return YXML_OK; - } - if(ch == (unsigned char)'\'' || ch == (unsigned char)'"') { - x->state = YXMLS_dt1; - x->quote = ch; - x->nextstate = YXMLS_dt0; - return YXML_OK; - } - if(ch == (unsigned char)'<') { - x->state = YXMLS_dt2; - return YXML_OK; - } - if(yxml_isChar(ch)) - return YXML_OK; - break; - case YXMLS_dt1: - if(x->quote == ch) { - x->state = x->nextstate; - return YXML_OK; - } - if(yxml_isChar(ch)) - return YXML_OK; - break; - case YXMLS_dt2: - if(ch == (unsigned char)'?') { - x->state = YXMLS_pi0; - x->nextstate = YXMLS_dt0; - return YXML_OK; - } - if(ch == (unsigned char)'!') { - x->state = YXMLS_dt3; - return YXML_OK; - } - break; - case YXMLS_dt3: - if(ch == (unsigned char)'-') { - x->state = YXMLS_comment1; - x->nextstate = YXMLS_dt0; - return YXML_OK; - } - if(yxml_isChar(ch)) { - x->state = YXMLS_dt4; - return YXML_OK; - } - break; - case YXMLS_dt4: - if(ch == (unsigned char)'\'' || ch == (unsigned char)'"') { - x->state = YXMLS_dt1; - x->quote = ch; - x->nextstate = YXMLS_dt4; - return YXML_OK; - } - if(ch == (unsigned char)'>') { - x->state = YXMLS_dt0; - return YXML_OK; - } - if(yxml_isChar(ch)) - return YXML_OK; - break; - case YXMLS_elem0: - if(yxml_isName(ch)) - return yxml_elemname(x, ch); - if(yxml_isSP(ch)) { - x->state = YXMLS_elem1; - return yxml_elemnameend(x, ch); - } - if(ch == (unsigned char)'/') { - x->state = YXMLS_elem3; - return yxml_elemnameend(x, ch); - } - if(ch == (unsigned char)'>') { - x->state = YXMLS_misc2; - return yxml_elemnameend(x, ch); - } - break; - case YXMLS_elem1: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'/') { - x->state = YXMLS_elem3; - return YXML_OK; - } - if(ch == (unsigned char)'>') { - x->state = YXMLS_misc2; - return YXML_OK; - } - if(yxml_isNameStart(ch)) { - x->state = YXMLS_attr0; - return yxml_attrstart(x, ch); - } - break; - case YXMLS_elem2: - if(yxml_isSP(ch)) { - x->state = YXMLS_elem1; - return YXML_OK; - } - if(ch == (unsigned char)'/') { - x->state = YXMLS_elem3; - return YXML_OK; - } - if(ch == (unsigned char)'>') { - x->state = YXMLS_misc2; - return YXML_OK; - } - break; - case YXMLS_elem3: - if(ch == (unsigned char)'>') { - x->state = YXMLS_misc2; - return yxml_selfclose(x, ch); - } - break; - case YXMLS_enc0: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'=') { - x->state = YXMLS_enc1; - return YXML_OK; - } - break; - case YXMLS_enc1: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'\'' || ch == (unsigned char)'"') { - x->state = YXMLS_enc2; - x->quote = ch; - return YXML_OK; - } - break; - case YXMLS_enc2: - if(yxml_isAlpha(ch)) { - x->state = YXMLS_enc3; - return YXML_OK; - } - break; - case YXMLS_enc3: - if(yxml_isEncName(ch)) - return YXML_OK; - if(x->quote == ch) { - x->state = YXMLS_xmldecl6; - return YXML_OK; - } - break; - case YXMLS_etag0: - if(yxml_isNameStart(ch)) { - x->state = YXMLS_etag1; - return yxml_elemclose(x, ch); - } - break; - case YXMLS_etag1: - if(yxml_isName(ch)) - return yxml_elemclose(x, ch); - if(yxml_isSP(ch)) { - x->state = YXMLS_etag2; - return yxml_elemcloseend(x, ch); - } - if(ch == (unsigned char)'>') { - x->state = YXMLS_misc2; - return yxml_elemcloseend(x, ch); - } - break; - case YXMLS_etag2: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'>') { - x->state = YXMLS_misc2; - return YXML_OK; - } - break; - case YXMLS_init: - if(ch == (unsigned char)'\xef') { - x->state = YXMLS_string; - x->nextstate = YXMLS_misc0; - x->string = (unsigned char *)"\xbb\xbf"; - return YXML_OK; - } - if(yxml_isSP(ch)) { - x->state = YXMLS_misc0; - return YXML_OK; - } - if(ch == (unsigned char)'<') { - x->state = YXMLS_le0; - return YXML_OK; - } - break; - case YXMLS_le0: - if(ch == (unsigned char)'!') { - x->state = YXMLS_lee1; - return YXML_OK; - } - if(ch == (unsigned char)'?') { - x->state = YXMLS_leq0; - return YXML_OK; - } - if(yxml_isNameStart(ch)) { - x->state = YXMLS_elem0; - return yxml_elemstart(x, ch); - } - break; - case YXMLS_le1: - if(ch == (unsigned char)'!') { - x->state = YXMLS_lee1; - return YXML_OK; - } - if(ch == (unsigned char)'?') { - x->state = YXMLS_pi0; - x->nextstate = YXMLS_misc1; - return YXML_OK; - } - if(yxml_isNameStart(ch)) { - x->state = YXMLS_elem0; - return yxml_elemstart(x, ch); - } - break; - case YXMLS_le2: - if(ch == (unsigned char)'!') { - x->state = YXMLS_lee2; - return YXML_OK; - } - if(ch == (unsigned char)'?') { - x->state = YXMLS_pi0; - x->nextstate = YXMLS_misc2; - return YXML_OK; - } - if(ch == (unsigned char)'/') { - x->state = YXMLS_etag0; - return YXML_OK; - } - if(yxml_isNameStart(ch)) { - x->state = YXMLS_elem0; - return yxml_elemstart(x, ch); - } - break; - case YXMLS_le3: - if(ch == (unsigned char)'!') { - x->state = YXMLS_comment0; - x->nextstate = YXMLS_misc3; - return YXML_OK; - } - if(ch == (unsigned char)'?') { - x->state = YXMLS_pi0; - x->nextstate = YXMLS_misc3; - return YXML_OK; - } - break; - case YXMLS_lee1: - if(ch == (unsigned char)'-') { - x->state = YXMLS_comment1; - x->nextstate = YXMLS_misc1; - return YXML_OK; - } - if(ch == (unsigned char)'D') { - x->state = YXMLS_string; - x->nextstate = YXMLS_dt0; - x->string = (unsigned char *)"OCTYPE"; - return YXML_OK; - } - break; - case YXMLS_lee2: - if(ch == (unsigned char)'-') { - x->state = YXMLS_comment1; - x->nextstate = YXMLS_misc2; - return YXML_OK; - } - if(ch == (unsigned char)'[') { - x->state = YXMLS_string; - x->nextstate = YXMLS_cd0; - x->string = (unsigned char *)"CDATA["; - return YXML_OK; - } - break; - case YXMLS_leq0: - if(ch == (unsigned char)'x') { - x->state = YXMLS_xmldecl0; - x->nextstate = YXMLS_misc1; - return yxml_pistart(x, ch); - } - if(yxml_isNameStart(ch)) { - x->state = YXMLS_pi1; - x->nextstate = YXMLS_misc1; - return yxml_pistart(x, ch); - } - break; - case YXMLS_misc0: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'<') { - x->state = YXMLS_le0; - return YXML_OK; - } - break; - case YXMLS_misc1: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'<') { - x->state = YXMLS_le1; - return YXML_OK; - } - break; - case YXMLS_misc2: - if(ch == (unsigned char)'<') { - x->state = YXMLS_le2; - return YXML_OK; - } - if(ch == (unsigned char)'&') { - x->state = YXMLS_misc2a; - return yxml_refstart(x, ch); - } - if(yxml_isChar(ch)) - return yxml_datacontent(x, ch); - break; - case YXMLS_misc2a: - if(yxml_isRef(ch)) - return yxml_ref(x, ch); - if(ch == (unsigned char)'\x3b') { - x->state = YXMLS_misc2; - return yxml_refcontent(x, ch); - } - break; - case YXMLS_misc3: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'<') { - x->state = YXMLS_le3; - return YXML_OK; - } - break; - case YXMLS_pi0: - if(yxml_isNameStart(ch)) { - x->state = YXMLS_pi1; - return yxml_pistart(x, ch); - } - break; - case YXMLS_pi1: - if(yxml_isName(ch)) - return yxml_piname(x, ch); - if(ch == (unsigned char)'?') { - x->state = YXMLS_pi4; - return yxml_pinameend(x, ch); - } - if(yxml_isSP(ch)) { - x->state = YXMLS_pi2; - return yxml_pinameend(x, ch); - } - break; - case YXMLS_pi2: - if(ch == (unsigned char)'?') { - x->state = YXMLS_pi3; - return YXML_OK; - } - if(yxml_isChar(ch)) - return yxml_datapi1(x, ch); - break; - case YXMLS_pi3: - if(ch == (unsigned char)'>') { - x->state = x->nextstate; - return yxml_pivalend(x, ch); - } - if(yxml_isChar(ch)) { - x->state = YXMLS_pi2; - return yxml_datapi2(x, ch); - } - break; - case YXMLS_pi4: - if(ch == (unsigned char)'>') { - x->state = x->nextstate; - return yxml_pivalend(x, ch); - } - break; - case YXMLS_std0: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'=') { - x->state = YXMLS_std1; - return YXML_OK; - } - break; - case YXMLS_std1: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'\'' || ch == (unsigned char)'"') { - x->state = YXMLS_std2; - x->quote = ch; - return YXML_OK; - } - break; - case YXMLS_std2: - if(ch == (unsigned char)'y') { - x->state = YXMLS_string; - x->nextstate = YXMLS_std3; - x->string = (unsigned char *)"es"; - return YXML_OK; - } - if(ch == (unsigned char)'n') { - x->state = YXMLS_string; - x->nextstate = YXMLS_std3; - x->string = (unsigned char *)"o"; - return YXML_OK; - } - break; - case YXMLS_std3: - if(x->quote == ch) { - x->state = YXMLS_xmldecl8; - return YXML_OK; - } - break; - case YXMLS_ver0: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'=') { - x->state = YXMLS_ver1; - return YXML_OK; - } - break; - case YXMLS_ver1: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'\'' || ch == (unsigned char)'"') { - x->state = YXMLS_string; - x->quote = ch; - x->nextstate = YXMLS_ver2; - x->string = (unsigned char *)"1."; - return YXML_OK; - } - break; - case YXMLS_ver2: - if(yxml_isNum(ch)) { - x->state = YXMLS_ver3; - return YXML_OK; - } - break; - case YXMLS_ver3: - if(yxml_isNum(ch)) - return YXML_OK; - if(x->quote == ch) { - x->state = YXMLS_xmldecl4; - return YXML_OK; - } - break; - case YXMLS_xmldecl0: - if(ch == (unsigned char)'m') { - x->state = YXMLS_xmldecl1; - return yxml_piname(x, ch); - } - if(yxml_isName(ch)) { - x->state = YXMLS_pi1; - return yxml_piname(x, ch); - } - if(ch == (unsigned char)'?') { - x->state = YXMLS_pi4; - return yxml_pinameend(x, ch); - } - if(yxml_isSP(ch)) { - x->state = YXMLS_pi2; - return yxml_pinameend(x, ch); - } - break; - case YXMLS_xmldecl1: - if(ch == (unsigned char)'l') { - x->state = YXMLS_xmldecl2; - return yxml_piname(x, ch); - } - if(yxml_isName(ch)) { - x->state = YXMLS_pi1; - return yxml_piname(x, ch); - } - if(ch == (unsigned char)'?') { - x->state = YXMLS_pi4; - return yxml_pinameend(x, ch); - } - if(yxml_isSP(ch)) { - x->state = YXMLS_pi2; - return yxml_pinameend(x, ch); - } - break; - case YXMLS_xmldecl2: - if(yxml_isSP(ch)) { - x->state = YXMLS_xmldecl3; - return yxml_piabort(x, ch); - } - if(yxml_isName(ch)) { - x->state = YXMLS_pi1; - return yxml_piname(x, ch); - } - break; - case YXMLS_xmldecl3: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'v') { - x->state = YXMLS_string; - x->nextstate = YXMLS_ver0; - x->string = (unsigned char *)"ersion"; - return YXML_OK; - } - break; - case YXMLS_xmldecl4: - if(yxml_isSP(ch)) { - x->state = YXMLS_xmldecl5; - return YXML_OK; - } - if(ch == (unsigned char)'?') { - x->state = YXMLS_xmldecl9; - return YXML_OK; - } - break; - case YXMLS_xmldecl5: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'?') { - x->state = YXMLS_xmldecl9; - return YXML_OK; - } - if(ch == (unsigned char)'e') { - x->state = YXMLS_string; - x->nextstate = YXMLS_enc0; - x->string = (unsigned char *)"ncoding"; - return YXML_OK; - } - if(ch == (unsigned char)'s') { - x->state = YXMLS_string; - x->nextstate = YXMLS_std0; - x->string = (unsigned char *)"tandalone"; - return YXML_OK; - } - break; - case YXMLS_xmldecl6: - if(yxml_isSP(ch)) { - x->state = YXMLS_xmldecl7; - return YXML_OK; - } - if(ch == (unsigned char)'?') { - x->state = YXMLS_xmldecl9; - return YXML_OK; - } - break; - case YXMLS_xmldecl7: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'?') { - x->state = YXMLS_xmldecl9; - return YXML_OK; - } - if(ch == (unsigned char)'s') { - x->state = YXMLS_string; - x->nextstate = YXMLS_std0; - x->string = (unsigned char *)"tandalone"; - return YXML_OK; - } - break; - case YXMLS_xmldecl8: - if(yxml_isSP(ch)) - return YXML_OK; - if(ch == (unsigned char)'?') { - x->state = YXMLS_xmldecl9; - return YXML_OK; - } - break; - case YXMLS_xmldecl9: - if(ch == (unsigned char)'>') { - x->state = YXMLS_misc1; - return YXML_OK; - } - break; - } - return YXML_ESYN; -} - - -yxml_ret_t yxml_eof(yxml_t *x) { - if(x->state != YXMLS_misc3) - return YXML_EEOF; - return YXML_OK; -} - - -/* vim: set noet sw=4 ts=4: */ diff --git a/external/yxml/yxml.h b/external/yxml/yxml.h deleted file mode 100644 index a0cc5f9d70..0000000000 --- a/external/yxml/yxml.h +++ /dev/null @@ -1,162 +0,0 @@ -/* Copyright (c) 2013-2014 Yoran Heling - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#ifndef YXML_H -#define YXML_H - -#include -#include - -#if defined(_MSC_VER) && !defined(__cplusplus) && !defined(inline) -#define inline __inline -#endif - -/* Full API documentation for this library can be found in the "yxml.md" file - * in the yxml git repository, or online at http://dev.yorhel.nl/yxml/man */ - -typedef enum { - YXML_EEOF = -5, /* Unexpected EOF */ - YXML_EREF = -4, /* Invalid character or entity reference (&whatever;) */ - YXML_ECLOSE = -3, /* Close tag does not match open tag ( .. ) */ - YXML_ESTACK = -2, /* Stack overflow (too deeply nested tags or too long element/attribute name) */ - YXML_ESYN = -1, /* Syntax error (unexpected byte) */ - YXML_OK = 0, /* Character consumed, no new token present */ - YXML_ELEMSTART = 1, /* Start of an element: '' or '' */ - YXML_ATTRSTART = 4, /* Attribute: 'Name=..' */ - YXML_ATTRVAL = 5, /* Attribute value */ - YXML_ATTREND = 6, /* End of attribute '.."' */ - YXML_PISTART = 7, /* Start of a processing instruction */ - YXML_PICONTENT = 8, /* Content of a PI */ - YXML_PIEND = 9 /* End of a processing instruction */ -} yxml_ret_t; - -/* When, exactly, are tokens returned? - * - * ' ELEMSTART - * '/' ELEMSTART, '>' ELEMEND - * ' ' ELEMSTART - * '>' - * '/', '>' ELEMEND - * Attr - * '=' ATTRSTART - * "X ATTRVAL - * 'Y' ATTRVAL - * 'Z' ATTRVAL - * '"' ATTREND - * '>' - * '/', '>' ELEMEND - * - * ' ELEMEND - */ - - -typedef struct { - /* PUBLIC (read-only) */ - - /* Name of the current element, zero-length if not in any element. Changed - * after YXML_ELEMSTART. The pointer will remain valid up to and including - * the next non-YXML_ATTR* token, the pointed-to buffer will remain valid - * up to and including the YXML_ELEMEND for the corresponding element. */ - char *elem; - - /* The last read character(s) of an attribute value (YXML_ATTRVAL), element - * data (YXML_CONTENT), or processing instruction (YXML_PICONTENT). Changed - * after one of the respective YXML_ values is returned, and only valid - * until the next yxml_parse() call. Usually, this string only consists of - * a single byte, but multiple bytes are returned in the following cases: - * - "": The two characters "?x" - * - "": The two characters "]x" - * - "": The three characters "]]x" - * - "&#N;" and "&#xN;", where dec(n) > 127. The referenced Unicode - * character is then encoded in multiple UTF-8 bytes. - */ - char data[8]; - - /* Name of the current attribute. Changed after YXML_ATTRSTART, valid up to - * and including the next YXML_ATTREND. */ - char *attr; - - /* Name/target of the current processing instruction, zero-length if not in - * a PI. Changed after YXML_PISTART, valid up to (but excluding) - * the next YXML_PIEND. */ - char *pi; - - /* Line number, byte offset within that line, and total bytes read. These - * values refer to the position _after_ the last byte given to - * yxml_parse(). These are useful for debugging and error reporting. */ - uint64_t byte; - uint64_t total; - uint32_t line; - - - /* PRIVATE */ - int state; - unsigned char *stack; /* Stack of element names + attribute/PI name, separated by \0. Also starts with a \0. */ - size_t stacksize, stacklen; - unsigned reflen; - unsigned quote; - int nextstate; /* Used for '@' state remembering and for the "string" consuming state */ - unsigned ignore; - unsigned char *string; -} yxml_t; - - -#ifdef __cplusplus -extern "C" { -#endif - -void yxml_init(yxml_t *, void *, size_t); - - -yxml_ret_t yxml_parse(yxml_t *, int); - - -/* May be called after the last character has been given to yxml_parse(). - * Returns YXML_OK if the XML document is valid, YXML_EEOF otherwise. Using - * this function isn't really necessary, but can be used to detect documents - * that don't end correctly. In particular, an error is returned when the XML - * document did not contain a (complete) root element, or when the document - * ended while in a comment or processing instruction. */ -yxml_ret_t yxml_eof(yxml_t *); - -#ifdef __cplusplus -} -#endif - - -/* Returns the length of the element name (x->elem), attribute name (x->attr), - * or PI name (x->pi). This function should ONLY be used directly after the - * YXML_ELEMSTART, YXML_ATTRSTART or YXML_PISTART (respectively) tokens have - * been returned by yxml_parse(), calling this at any other time may not give - * the correct results. This function should also NOT be used on strings other - * than x->elem, x->attr or x->pi. */ -static inline size_t yxml_symlen(yxml_t *x, const char *s) { - return (x->stack + x->stacklen) - (const unsigned char*)s; -} - -#endif - -/* vim: set noet sw=4 ts=4: */ diff --git a/libpkg/Makefile.in b/libpkg/Makefile.in index 517a8aa0c2..f93db4d6fd 100644 --- a/libpkg/Makefile.in +++ b/libpkg/Makefile.in @@ -71,7 +71,6 @@ LOCAL_CFLAGS= -I$(top_srcdir)/compat \ -I$(top_srcdir)/external/sqlite \ -I$(top_srcdir)/external/uthash \ -I$(top_srcdir)/external/picosat \ - -I$(top_srcdir)/external/yxml \ -I$(top_srcdir)/external/uthash \ -I$(top_srcdir)/external/include \ -I$(top_srcdir)/external/libucl/include \ @@ -101,7 +100,6 @@ LOCAL_LDFLAGS= @waflags@ \ -L$(top_builddir)/external/libucl -lucl_pic \ -L$(top_builddir)/external/libpicosat -lpicosat_pic \ -L$(top_builddir)/external/liblinenoise -llinenoise_pic \ - -L$(top_builddir)/external/libyxml -lyxml_pic \ -L$(top_builddir)/external/libblake2 -lblake2_pic \ -L$(top_builddir)/compat -lbsd_compat_pic \ -L$(top_builddir)/external/liblua -llua_pic \ @@ -116,7 +114,6 @@ STATIC_LIBS= @REPOS_STATIC_LIBS@ \ $(top_builddir)/external/libucl/libucl.a \ $(top_builddir)/external/libpicosat/libpicosat.a \ $(top_builddir)/external/liblinenoise/liblinenoise.a \ - $(top_builddir)/external/libyxml/libyxml.a \ $(top_builddir)/external/libblake2/libblake2.a \ $(top_builddir)/compat/libbsd_compat.a \ $(top_builddir)/external/liblua/liblua.a \ diff --git a/libpkg/pkg_audit.c b/libpkg/pkg_audit.c index 88a59dc5db..bee563dc14 100644 --- a/libpkg/pkg_audit.c +++ b/libpkg/pkg_audit.c @@ -41,8 +41,7 @@ #include #include #include - -#include +#include #ifdef __linux__ # ifdef __GLIBC__ @@ -53,6 +52,7 @@ #include "pkg.h" #include "pkg/audit.h" #include "private/pkg.h" +#include #include "private/event.h" #define PKG_AUDIT_MAX_DOWNLOAD_SIZE (64LL * 1024 * 1024) @@ -86,7 +86,7 @@ * multiple vulnerabilities, it could be a large win. */ struct pkg_audit_item { - struct pkg_audit_entry *e; /* Entry itself */ + struct pkg_osvf_entry *e; /* Entry itself */ size_t pkg_idx; /* Index into e->packages */ size_t name_idx; /* Index into e->packages.d[pkg_idx].names */ size_t noglob_len; /* Prefix without glob characters */ @@ -95,8 +95,9 @@ struct pkg_audit_item { }; struct pkg_audit { - audit_entryv_t entries; + osvf_entryv_t entries; struct pkg_audit_item *items; + struct ucl_parser *parser; bool parsed; bool loaded; char **ignore_globs; @@ -119,33 +120,10 @@ struct pkg_audit { static size_t audit_entry_first_byte_idx[256]; static void -pkg_audit_free_entry(struct pkg_audit_entry *e) -{ - vec_foreach(e->packages, pi) { - struct pkg_audit_package *p = &e->packages.d[pi]; - vec_foreach(p->names, ni) - free(p->names.d[ni].pkgname); - vec_free(&p->names); - vec_foreach(p->versions, vi) { - free(p->versions.d[vi].v1.version); - free(p->versions.d[vi].v2.version); - } - vec_free(&p->versions); - } - vec_free(&e->packages); - vec_foreach(e->cve, ci) - free(e->cve.d[ci].cvename); - vec_free(&e->cve); - free(e->url); - free(e->desc); - free(e->id); -} - -static void -pkg_audit_free_entries(audit_entryv_t *entries) +pkg_audit_free_entries(osvf_entryv_t *entries) { vec_foreach(*entries, i) - pkg_audit_free_entry(&entries->d[i]); + pkg_osvf_free_entry(entries->d[i]); vec_free(entries); } @@ -315,8 +293,14 @@ pkg_audit_fetch(const char *src, const char *dest) } }; + if(!pkg_config_get("OSVF_SITE")) + { + pkg_emit_notice("There is not OSVF_SITE config key available. Can't continue"); + return retcode; + } + if (src == NULL) { - src = pkg_object_string(pkg_config_get("VULNXML_SITE")); + src = pkg_object_string(pkg_config_get("OSVF_SITE")); } tmpdir = getenv("TMPDIR"); @@ -324,9 +308,9 @@ pkg_audit_fetch(const char *src, const char *dest) tmpdir = "/tmp"; strlcpy(tmp, tmpdir, sizeof(tmp)); - strlcat(tmp, "/vuln.xml.XXXXXXXXXX", sizeof(tmp)); + strlcat(tmp, "/freebsd-osv.json.XXXXXXXXXX", sizeof(tmp)); if (pkg_audit_get_path_limit(tmpdir, PKG_AUDIT_MAX_DOWNLOAD_SIZE, - &download_limit) != EPKG_OK) + &download_limit) != EPKG_OK) goto cleanup; if (dest != NULL) { @@ -334,7 +318,7 @@ pkg_audit_fetch(const char *src, const char *dest) t = st.st_mtime; } else { dfd = pkg_get_dbdirfd(); - if (fstatat(dfd, "vuln.xml", &st, 0) != -1) + if (fstatat(dfd, "freebsd-osv.json", &st, 0) != -1) t = st.st_mtime; } @@ -343,11 +327,11 @@ pkg_audit_fetch(const char *src, const char *dest) case EPKG_OK: break; case EPKG_UPTODATE: - pkg_emit_notice("vulnxml file up-to-date"); + pkg_emit_notice("OSVF database file up-to-date"); retcode = EPKG_OK; goto cleanup; default: - pkg_emit_error("cannot fetch vulnxml file"); + pkg_emit_error("cannot fetch OSVF database file"); goto cleanup; } /* Create a temporary output beside the final database. */ @@ -355,7 +339,7 @@ pkg_audit_fetch(const char *src, const char *dest) hidden_tempfile(outtmp, sizeof(outtmp), dest); outfd = mkstemp(outtmp); } else { - hidden_tempfile(outname, sizeof(outname), "vuln.xml"); + hidden_tempfile(outname, sizeof(outname), "freebsd-osv.json"); outfd = openat(dfd, outname, O_CREAT|O_EXCL|O_RDWR, 0600); } if (outfd == -1) { @@ -369,7 +353,7 @@ pkg_audit_fetch(const char *src, const char *dest) cbdata.fname = tmp; cbdata.out = outfd; - cbdata.dest = dest != NULL ? dest : "vuln.xml"; + cbdata.dest = dest != NULL ? dest : "freebsd-osv.json"; cbdata.max_size = extract_limit; if (fstat(fd, &st) == -1) { pkg_emit_errno("fstat", "audit database download"); @@ -401,7 +385,7 @@ pkg_audit_fetch(const char *src, const char *dest) } outfd = -1; if ((dest != NULL && rename(outtmp, dest) == -1) || - (dest == NULL && renameat(dfd, outname, dfd, "vuln.xml") == -1)) { + (dest == NULL && renameat(dfd, outname, dfd, "freebsd-osv.json") == -1)) { pkg_emit_errno("rename", cbdata.dest); retcode = EPKG_FATAL; goto cleanup; @@ -423,287 +407,6 @@ pkg_audit_fetch(const char *src, const char *dest) return (retcode); } -enum vulnxml_parse_state { - VULNXML_PARSE_INIT = 0, - VULNXML_PARSE_VULN, - VULNXML_PARSE_TOPIC, - VULNXML_PARSE_PACKAGE, - VULNXML_PARSE_PACKAGE_NAME, - VULNXML_PARSE_RANGE, - VULNXML_PARSE_RANGE_GT, - VULNXML_PARSE_RANGE_GE, - VULNXML_PARSE_RANGE_LT, - VULNXML_PARSE_RANGE_LE, - VULNXML_PARSE_RANGE_EQ, - VULNXML_PARSE_CVE -}; - -enum vulnxml_parse_attribute_state { - VULNXML_ATTR_NONE = 0, - VULNXML_ATTR_VID, -}; - -struct vulnxml_userdata { - struct pkg_audit_entry *cur_entry; - struct pkg_audit *audit; - enum vulnxml_parse_state state; - sb_t content; - int range_num; - enum vulnxml_parse_attribute_state attr; -}; - -static void -vulnxml_start_element(struct vulnxml_userdata *ud, yxml_t *xml) -{ - if (ud->state == VULNXML_PARSE_INIT && STRIEQ(xml->elem, "vuln")) { - vec_push(&ud->audit->entries, ((struct pkg_audit_entry){0})); - ud->cur_entry = &ud->audit->entries.d[ud->audit->entries.len - 1]; - ud->state = VULNXML_PARSE_VULN; - } - else if (ud->state == VULNXML_PARSE_VULN && STRIEQ(xml->elem, "topic")) { - ud->state = VULNXML_PARSE_TOPIC; - } - else if (ud->state == VULNXML_PARSE_VULN && STRIEQ(xml->elem, "package")) { - vec_push(&ud->cur_entry->packages, ((struct pkg_audit_package){0})); - ud->state = VULNXML_PARSE_PACKAGE; - } - else if (ud->state == VULNXML_PARSE_VULN && STRIEQ(xml->elem, "cvename")) { - ud->state = VULNXML_PARSE_CVE; - } - else if (ud->state == VULNXML_PARSE_PACKAGE && STRIEQ(xml->elem, "name")) { - ud->state = VULNXML_PARSE_PACKAGE_NAME; - struct pkg_audit_package *cur_pkg = - &ud->cur_entry->packages.d[ud->cur_entry->packages.len - 1]; - vec_push(&cur_pkg->names, ((struct pkg_audit_pkgname){0})); - } - else if (ud->state == VULNXML_PARSE_PACKAGE && STRIEQ(xml->elem, "range")) { - ud->state = VULNXML_PARSE_RANGE; - struct pkg_audit_package *cur_pkg = - &ud->cur_entry->packages.d[ud->cur_entry->packages.len - 1]; - vec_push(&cur_pkg->versions, ((struct pkg_audit_versions_range){0})); - ud->range_num = 0; - } - else if (ud->state == VULNXML_PARSE_RANGE && STRIEQ(xml->elem, "gt")) { - ud->range_num ++; - ud->state = VULNXML_PARSE_RANGE_GT; - } - else if (ud->state == VULNXML_PARSE_RANGE && STRIEQ(xml->elem, "ge")) { - ud->range_num ++; - ud->state = VULNXML_PARSE_RANGE_GE; - } - else if (ud->state == VULNXML_PARSE_RANGE && STRIEQ(xml->elem, "lt")) { - ud->range_num ++; - ud->state = VULNXML_PARSE_RANGE_LT; - } - else if (ud->state == VULNXML_PARSE_RANGE && STRIEQ(xml->elem, "le")) { - ud->range_num ++; - ud->state = VULNXML_PARSE_RANGE_LE; - } - else if (ud->state == VULNXML_PARSE_RANGE && STRIEQ(xml->elem, "eq")) { - ud->range_num ++; - ud->state = VULNXML_PARSE_RANGE_EQ; - } -} - -static void -vulnxml_end_element(struct vulnxml_userdata *ud, yxml_t *xml) -{ - struct pkg_audit_versions_range *vers; - int range_type = -1; - - - if (ud->state == VULNXML_PARSE_VULN && STRIEQ(xml->elem, "vuxml")) { - /* Entry is already in the vec, just validate it */ - struct pkg_audit_entry *e = ud->cur_entry; - if (e->packages.len == 0 || - (e->packages.len > 0 && e->packages.d[0].names.len == 0)) { - /* Invalid entry: remove from vec and free */ - pkg_audit_free_entry(e); - ud->audit->entries.len--; - } - ud->state = VULNXML_PARSE_INIT; - } - else if (ud->state == VULNXML_PARSE_TOPIC && STRIEQ(xml->elem, "vuln")) { - ud->cur_entry->desc = xstrdup(sb_str(&ud->content)); - ud->state = VULNXML_PARSE_VULN; - } - else if (ud->state == VULNXML_PARSE_CVE && STRIEQ(xml->elem, "references")) { - vec_push(&ud->cur_entry->cve, - ((struct pkg_audit_cve){ .cvename = xstrdup(sb_str(&ud->content)) })); - ud->state = VULNXML_PARSE_VULN; - } - else if (ud->state == VULNXML_PARSE_PACKAGE && STRIEQ(xml->elem, "affects")) { - ud->state = VULNXML_PARSE_VULN; - } - else if (ud->state == VULNXML_PARSE_PACKAGE_NAME && STRIEQ(xml->elem, "package")) { - struct pkg_audit_package *cur_pkg = - &ud->cur_entry->packages.d[ud->cur_entry->packages.len - 1]; - struct pkg_audit_pkgname *cur_name = - &cur_pkg->names.d[cur_pkg->names.len - 1]; - cur_name->pkgname = xstrdup(sb_str(&ud->content)); - ud->state = VULNXML_PARSE_PACKAGE; - } - else if (ud->state == VULNXML_PARSE_RANGE && STRIEQ(xml->elem, "package")) { - ud->state = VULNXML_PARSE_PACKAGE; - } - else if (ud->state == VULNXML_PARSE_RANGE_GT && STRIEQ(xml->elem, "range")) { - range_type = GT; - ud->state = VULNXML_PARSE_RANGE; - } - else if (ud->state == VULNXML_PARSE_RANGE_GE && STRIEQ(xml->elem, "range")) { - range_type = GTE; - ud->state = VULNXML_PARSE_RANGE; - } - else if (ud->state == VULNXML_PARSE_RANGE_LT && STRIEQ(xml->elem, "range")) { - range_type = LT; - ud->state = VULNXML_PARSE_RANGE; - } - else if (ud->state == VULNXML_PARSE_RANGE_LE && STRIEQ(xml->elem, "range")) { - range_type = LTE; - ud->state = VULNXML_PARSE_RANGE; - } - else if (ud->state == VULNXML_PARSE_RANGE_EQ && STRIEQ(xml->elem, "range")) { - range_type = EQ; - ud->state = VULNXML_PARSE_RANGE; - } - - if (range_type > 0) { - struct pkg_audit_package *cur_pkg = - &ud->cur_entry->packages.d[ud->cur_entry->packages.len - 1]; - vers = &cur_pkg->versions.d[cur_pkg->versions.len - 1]; - if (ud->range_num == 1) { - vers->v1.version = xstrdup(sb_str(&ud->content)); - vers->v1.type = range_type; - } - else if (ud->range_num == 2) { - vers->v2.version = xstrdup(sb_str(&ud->content)); - vers->v2.type = range_type; - } - } - sb_reset(&ud->content); -} - -static void -vulnxml_start_attribute(struct vulnxml_userdata *ud, yxml_t *xml) -{ - if (ud->state != VULNXML_PARSE_VULN) - return; - - if (STRIEQ(xml->attr, "vid")) - ud->attr = VULNXML_ATTR_VID; -} - -static void -vulnxml_end_attribute(struct vulnxml_userdata *ud, yxml_t *xml __unused) -{ - - if (ud->state == VULNXML_PARSE_VULN && ud->attr == VULNXML_ATTR_VID) { - ud->cur_entry->id = xstrdup(sb_str(&ud->content)); - ud->attr = VULNXML_ATTR_NONE; - } - sb_reset(&ud->content); -} - -static void -vulnxml_val_attribute(struct vulnxml_userdata *ud, yxml_t *xml) -{ - if (ud->state == VULNXML_PARSE_VULN && ud->attr == VULNXML_ATTR_VID) { - sb_cat(&ud->content, xml->data); - } -} - -static void -vulnxml_handle_data(struct vulnxml_userdata *ud, yxml_t *xml) -{ - - switch(ud->state) { - case VULNXML_PARSE_INIT: - case VULNXML_PARSE_VULN: - case VULNXML_PARSE_PACKAGE: - case VULNXML_PARSE_RANGE: - /* On these states we do not need any data */ - break; - case VULNXML_PARSE_TOPIC: - case VULNXML_PARSE_PACKAGE_NAME: - case VULNXML_PARSE_CVE: - case VULNXML_PARSE_RANGE_GT: - case VULNXML_PARSE_RANGE_GE: - case VULNXML_PARSE_RANGE_LT: - case VULNXML_PARSE_RANGE_LE: - case VULNXML_PARSE_RANGE_EQ: - sb_cat(&ud->content, xml->data); - break; - } -} - -static int -pkg_audit_parse_vulnxml(struct pkg_audit *audit) -{ - int ret = EPKG_FATAL; - yxml_t x; - yxml_ret_t r; - char buf[BUFSIZ]; - char *walk, *end; - struct vulnxml_userdata ud = { 0 }; - - yxml_init(&x, buf, BUFSIZ); - ud.audit = audit; - ud.state = VULNXML_PARSE_INIT; - - walk = audit->map; - end = walk + audit->len; - while (walk < end) { - r = yxml_parse(&x, *walk++); - switch (r) { - case YXML_EEOF: - case YXML_EREF: - case YXML_ESTACK: - pkg_emit_error("Unexpected EOF while parsing vulnxml"); - goto out; - case YXML_ESYN: - pkg_emit_error("Syntax error while parsing vulnxml"); - goto out; - case YXML_ECLOSE: - pkg_emit_error("Close tag does not match open tag line %d", x.line); - goto out; - case YXML_ELEMSTART: - vulnxml_start_element(&ud, &x); - break; - case YXML_ELEMEND: - vulnxml_end_element(&ud, &x); - break; - case YXML_CONTENT: - vulnxml_handle_data(&ud, &x); - break; - case YXML_ATTRVAL: - vulnxml_val_attribute(&ud, &x); - break; - case YXML_ATTRSTART: - vulnxml_start_attribute(&ud, &x); - break; - /* ignore */ - case YXML_ATTREND: - vulnxml_end_attribute(&ud, &x); - /* ignore */ - break; - case YXML_OK: - case YXML_PISTART: - case YXML_PICONTENT: - case YXML_PIEND: - break; - } - } - - if (yxml_eof(&x) == YXML_OK) - ret = EPKG_OK; - else - pkg_emit_error("Invalid end of XML"); -out: - sb_fini(&ud.content); - - return (ret); -} - /* * Returns the length of the largest prefix without globbing * characters, as per fnmatch(). @@ -725,7 +428,7 @@ pkg_audit_str_noglob_len(const char *s) static const char * pkg_audit_item_pkgname(const struct pkg_audit_item *item) { - return item->e->packages.d[item->pkg_idx].names.d[item->name_idx].pkgname; + return item->e->audit.packages.d[item->pkg_idx].names.d[item->name_idx].pkgname; } /* @@ -764,7 +467,7 @@ pkg_audit_entry_cmp(const void *a, const void *b) * next distinct prefix. */ static struct pkg_audit_item * -pkg_audit_preprocess(audit_entryv_t *entries) +pkg_audit_preprocess(osvf_entryv_t *entries) { struct pkg_audit_item *ret; size_t i, n, tofill; @@ -772,7 +475,8 @@ pkg_audit_preprocess(audit_entryv_t *entries) /* Count total items (entry x package x name) */ n = 0; vec_foreach(*entries, ei) { - struct pkg_audit_entry *e = &entries->d[ei]; + struct pkg_osvf_entry *osvf = entries->d[ei]; + struct pkg_audit_entry *e = &osvf->audit; vec_foreach(e->packages, pi) { n += e->packages.d[pi].names.len; } @@ -782,13 +486,14 @@ pkg_audit_preprocess(audit_entryv_t *entries) n = 0; vec_foreach(*entries, ei) { - struct pkg_audit_entry *e = &entries->d[ei]; + struct pkg_osvf_entry *osvf = entries->d[ei]; + struct pkg_audit_entry *e = &osvf->audit; vec_foreach(e->packages, pi) { struct pkg_audit_package *p = &e->packages.d[pi]; vec_foreach(p->names, ni) { const char *pkgname = p->names.d[ni].pkgname; if (pkgname != NULL) { - ret[n].e = e; + ret[n].e = osvf; ret[n].pkg_idx = pi; ret[n].name_idx = ni; ret[n].noglob_len = pkg_audit_str_noglob_len(pkgname); @@ -886,7 +591,7 @@ bool pkg_audit_is_vulnerable(struct pkg_audit *audit, struct pkg *pkg, struct pkg_audit_issues **ai, bool stop_quick) { - struct pkg_audit_entry *e; + struct pkg_osvf_entry *e; struct pkg_audit_item *a; bool res = false, res1, res2; @@ -930,11 +635,11 @@ pkg_audit_is_vulnerable(struct pkg_audit *audit, struct pkg *pkg, * Assume that all versions should be checked */ res = true; - pkg_audit_add_entry(e, a[i].pkg_idx, ai); + pkg_audit_add_entry(&e->audit, a[i].pkg_idx, ai); } else { audit_versv_t *versions = - &e->packages.d[a[i].pkg_idx].versions; + &e->audit.packages.d[a[i].pkg_idx].versions; vec_foreach(*versions, vi) { struct pkg_audit_versions_range *vers = &versions->d[vi]; @@ -945,7 +650,7 @@ pkg_audit_is_vulnerable(struct pkg_audit *audit, struct pkg *pkg, if (res1 && res2) { res = true; - pkg_audit_add_entry(e, + pkg_audit_add_entry(&e->audit, a[i].pkg_idx, ai); break; } @@ -967,6 +672,13 @@ pkg_audit_new(void) audit = xcalloc(1, sizeof(struct pkg_audit)); + if(!audit) + { + return NULL; + } + + audit->parser = ucl_parser_new(0); + return (audit); } @@ -974,7 +686,6 @@ int pkg_audit_load(struct pkg_audit *audit, const char *fname) { int dfd, fd; - void *mem; struct stat st; if (fname != NULL) { @@ -982,7 +693,7 @@ pkg_audit_load(struct pkg_audit *audit, const char *fname) return (EPKG_FATAL); } else { dfd = pkg_get_dbdirfd(); - if ((fd = openat(dfd, "vuln.xml", O_RDONLY)) == -1) + if ((fd = openat(dfd, "freebsd-osv.json", O_RDONLY)) == -1) return (EPKG_FATAL); } @@ -991,14 +702,19 @@ pkg_audit_load(struct pkg_audit *audit, const char *fname) return (EPKG_FATAL); } - if ((mem = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0)) == MAP_FAILED) { + /* + * Parse JSON which should be an array containing single + * OSV compatible vulnerability as an object + */ + if (!ucl_parser_add_fd(audit->parser, fd)) + { + pkg_emit_error("Error parsing UCL file '%s': %s'", + fname, ucl_parser_get_error(audit->parser)); close(fd); return (EPKG_FATAL); } close(fd); - audit->map = mem; - audit->len = st.st_size; audit->loaded = true; return (EPKG_OK); @@ -1008,17 +724,54 @@ pkg_audit_load(struct pkg_audit *audit, const char *fname) int pkg_audit_process(struct pkg_audit *audit) { + ucl_object_t *root_obj = NULL; + ucl_object_iter_t it = NULL; + const ucl_object_t *cur = NULL; + struct pkg_osvf_entry *cur_entry = NULL; + struct ucl_schema_error err; + if (geteuid() == 0) return (EPKG_FATAL); if (!audit->loaded) return (EPKG_FATAL); - if (pkg_audit_parse_vulnxml(audit) == EPKG_FATAL) + root_obj = ucl_parser_get_object(audit->parser); + ucl_parser_free(audit->parser); + audit->parser = NULL; + + if (root_obj == NULL) + { + pkg_emit_error("JSON cannot be parsed: %s", err.msg); return (EPKG_FATAL); + } + + if(root_obj && ucl_object_type(root_obj) == UCL_ARRAY) + { + while ((cur = ucl_iterate_object(root_obj, &it, true))) + { + if(cur && ucl_object_type(cur) == UCL_OBJECT) + { + cur_entry = pkg_osvf_create_entry((ucl_object_t *)cur); + + if(!cur_entry) + { + return (EPKG_FATAL); + } + + vec_push(&audit->entries, cur_entry); + } + } + } + else + { + pkg_emit_error("JSON root is not array"); + return (EPKG_FATAL); + } audit->items = pkg_audit_preprocess(&audit->entries); audit->parsed = true; + ucl_object_unref(root_obj); return (EPKG_OK); } @@ -1031,8 +784,9 @@ pkg_audit_free (struct pkg_audit *audit) pkg_audit_free_entries(&audit->entries); free(audit->items); } - if (audit->loaded) { - munmap(audit->map, audit->len); + if (audit->parser) { + ucl_parser_free(audit->parser); + audit->parser = NULL; } free(audit); } diff --git a/libpkg/pkg_config.c b/libpkg/pkg_config.c index df455d4db4..789080d373 100644 --- a/libpkg/pkg_config.c +++ b/libpkg/pkg_config.c @@ -61,6 +61,9 @@ #ifndef DEFAULT_VULNXML_URL #define DEFAULT_VULNXML_URL "https://vuxml.freebsd.org/freebsd/vuln.xml.xz" #endif +#ifndef DEFAULT_OSVF_URL +#define DEFAULT_OSVF_URL "https://raw.githubusercontent.com/illuusio/freebsd-osv/refs/heads/main/db/freebsd-osv.json" +#endif #ifdef OSMAJOR #define STRINGIFY(X) TEXT(X) @@ -174,7 +177,12 @@ static struct config_entry c[] = { { PKG_STRING, "VULNXML_SITE", - DEFAULT_VULNXML_URL, + DEFAULT_OSVF_URL, + }, + { + PKG_STRING, + "OSVF_SITE", + DEFAULT_OSVF_URL, }, { PKG_INT, diff --git a/libpkg/pkg_osvf.c b/libpkg/pkg_osvf.c index 44361dcaa8..2e8b8fee1b 100644 --- a/libpkg/pkg_osvf.c +++ b/libpkg/pkg_osvf.c @@ -893,6 +893,12 @@ pkg_osvf_parse_events(audit_versv_t *versions, const ucl_object_t *event_array, range_type = pkg_osvf_get_event(type); + /* Parses package structure from events: + { + "introduced|fixed|last_affected|limit": "1.0.0" + } + */ + while ((cur = ucl_iterate_object(event_array, &it, true))) { const char *key; diff --git a/libpkg/private/pkg_osvf.h b/libpkg/private/pkg_osvf.h index ae3c89ae1a..42bd9de1c3 100644 --- a/libpkg/private/pkg_osvf.h +++ b/libpkg/private/pkg_osvf.h @@ -92,6 +92,7 @@ struct pkg_osvf_entry { struct pkg_audit_entry audit; struct pkg_osvf_reference *references; }; +typedef vec_t(struct pkg_osvf_entry *) osvf_entryv_t; ucl_object_t * pkg_osvf_open(const char *location); diff --git a/scripts/completion/_pkg.in b/scripts/completion/_pkg.in index 04936f6afd..3fd484a078 100644 --- a/scripts/completion/_pkg.in +++ b/scripts/completion/_pkg.in @@ -157,7 +157,8 @@ _pkg_config_opts() { "UNSET_TIMESTAMP[don't include timestamps in package tar(1) archive]:boolean:(yes no)" \ 'VALID_URL_SCHEME[valid URL schemes]:list' \ 'VERSION_SOURCE[default database for comparing version numbers in pkg-version(8)]:database:(( I\:index P\:ports R\:remote ))' \ - 'VULNXML_SITE[specify URL to fetch vuln.xml vulnerability database from]: : _urls -F "( gopher:* file:* ftp:* )"' \ + 'VULNXML_SITE[specify URL to fetch freebsd-osv.json OSV JSON vulnerability database from. Please use OSVF_SITE for new installations!]: : _urls -F "( gopher:* file:* ftp:* )"' \ + 'OSVF_SITE[specify URL to fetch freebsd-osv.json OSV JSON vulnerability database from]: : _urls -F "( gopher:* file:* ftp:* )"' \ 'WARN_SIZE_LIMIT[ask user when performing changes for more than this limit]:limit [1048576]' \ 'WORKERS_COUNT[how many workers are used for pkg-repo]:workers:( {0..$(sysctl -n hw.ncpu)} )' } diff --git a/scripts/periodic/405.pkg-base-audit.in b/scripts/periodic/405.pkg-base-audit.in index 9c66485e92..1e4029bad5 100644 --- a/scripts/periodic/405.pkg-base-audit.in +++ b/scripts/periodic/405.pkg-base-audit.in @@ -49,7 +49,7 @@ fi # Compute PKG_DBDIR from the config file. pkgcmd=%prefix%/sbin/pkg PKG_DBDIR=`${pkgcmd} config PKG_DBDIR` -auditfile="${PKG_DBDIR}/vuln.xml" +auditfile="${PKG_DBDIR}/freebsd-osv.json" audit_base() { local pkgargs="$1" diff --git a/scripts/periodic/410.pkg-audit.in b/scripts/periodic/410.pkg-audit.in index a6b46cb410..f723db23e3 100755 --- a/scripts/periodic/410.pkg-audit.in +++ b/scripts/periodic/410.pkg-audit.in @@ -50,7 +50,7 @@ fi # Compute PKG_DBDIR from the config file. pkgcmd=%prefix%/sbin/pkg PKG_DBDIR=`${pkgcmd} config PKG_DBDIR` -auditfile="${PKG_DBDIR}/vuln.xml" +auditfile="${PKG_DBDIR}/freebsd-osv.json" audit_pkgs() { local pkgargs="$1" diff --git a/src/audit.c b/src/audit.c index d0c127fe65..a84c25b366 100644 --- a/src/audit.c +++ b/src/audit.c @@ -266,11 +266,11 @@ exec_audit(int argc, char **argv) if (pkg_audit_load(audit, audit_file) != EPKG_OK) { if (errno == ENOENT) - warnx("vulnxml file %s does not exist. " + warnx("FreeBSD OSV file %s does not exist. " "Try running 'pkg audit -F' first", - audit_file == NULL ? "vuln.xml" : audit_file); + audit_file == NULL ? "freebsd-osv.json" : audit_file); else - warn("unable to open vulnxml file %s", + warn("unable to open OSV JSON file %s", audit_file); pkg_audit_free(audit); @@ -386,7 +386,7 @@ exec_audit(int argc, char **argv) pkg_drop_privileges(); - /* Now we have vulnxml loaded and check list formed */ + /* Now we have OSV loaded and check list formed */ #ifdef HAVE_CAPSICUM #ifndef COVERAGE if (cap_enter() < 0 && errno != ENOSYS) { @@ -489,13 +489,13 @@ exec_audit(int argc, char **argv) } else { ucl_object_insert_key(top, ucl_object_fromint(vuln), "pkg_count", 9, false ); ucl_object_insert_key(top, vuln_objs, "packages", 8, false); - char *output = ucl_object_emit(top, raw); + unsigned char *output = ucl_object_emit(top, raw); fprintf(stdout, "%s\n", output); free(output); ucl_object_unref(top); } } else { - warnx("cannot process vulnxml"); + warnx("cannot process OSV"); ret = EXIT_FAILURE; hash_destroy(check); } diff --git a/src/pkg.conf.sample b/src/pkg.conf.sample index 405f6bc762..6745acb72e 100644 --- a/src/pkg.conf.sample +++ b/src/pkg.conf.sample @@ -25,7 +25,7 @@ #SYSLOG = true; #ABI = "freebsd:10:x86:64"; # Autogenerated #DEVELOPER_MODE = false; -#VULNXML_SITE = "http://vuxml.freebsd.org/freebsd/vuln.xml.xz"; +#OSVF_SITE = "https://raw.githubusercontent.com/illuusio/freebsd-osv/refs/heads/main/db/freebsd-osv.json"; #FETCH_RETRY = 3; #PKG_PLUGINS_DIR = "/usr/local/lib/pkg/"; #PKG_ENABLE_PLUGINS = true; diff --git a/tests/cocci/README.md b/tests/cocci/README.md index f6ded73bd9..0982e8d9e7 100644 --- a/tests/cocci/README.md +++ b/tests/cocci/README.md @@ -22,7 +22,7 @@ Usage From the pkg's source root (use _libpkg_ or _src_ as `$DIR`): % spatch -I . -I /usr/include -I /usr/local/include -I libpkg -I src \ - -I external/blake2 -I external/yxml -I external/include \ + -I external/blake2 -I external/include \ -I external/libelf -I external/libfetch \ -I external/libsbuf -I external/libucl/include -I external/linenoise \ -I external/picosat -I external/sqlite \ diff --git a/tests/frontend/audit.sh b/tests/frontend/audit.sh index 5492de0436..a88e3326b9 100644 --- a/tests/frontend/audit.sh +++ b/tests/frontend/audit.sh @@ -25,25 +25,53 @@ setup_packages() { # Helper: create a vuln XML with a vulnerability affecting test >=1.0 <2.0 create_vuln_db() { - cat > vuln.xml << 'EOF' - - - - Test vulnerability in test package - - - test - - 1.0 - 2.0 - - - - - CVE-2024-00001 - - - + cat > vuln.json << 'EOF' +[ + { + "affected": [ + { + "package": { + "ecosystem": "FreeBSD:ports", + "name": "test" + }, + "ranges": [ + { + "events": [ + { + "introduced": "1.0" + }, + { + "fixed": "2.0" + } + ], + "type": "ECOSYSTEM" + } + ] + } + ], + "database_specific": { + "discovery": "2024-01-01T00:00:00Z", + "references": { + "cvename": [ + "CVE-2024-00001" + ] + }, + "vid": "test-vuln-001" + }, + "details": "Example description\n", + "id": "FreeBSD-2024-0001", + "modified": "2024-01-01T00:00:00Z", + "published": "2024-01-01T00:00:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://cveawg.mitre.org/api/cve/CVE-2024-00001" + } + ], + "schema_version": "1.7.0", + "summary": "Test vulnerability in test package" + } +] EOF } @@ -58,7 +86,7 @@ audit_vulnerable_body() { -o match:"CVE-2024-00001" \ -o match:"1 problem" \ -s exit:1 \ - pkg audit -f vuln.xml + pkg audit -f vuln.json } audit_not_vulnerable_body() { @@ -71,54 +99,81 @@ audit_not_vulnerable_body() { atf_check \ -o match:"0 problem" \ -s exit:0 \ - pkg audit -f vuln.xml + pkg audit -f vuln.json } audit_empty_db_body() { setup_packages - cat > vuln.xml << 'EOF' - - - + cat > vuln.json << 'EOF' +[] EOF # Empty vuln db -> no problems atf_check \ -o match:"0 problem" \ -s exit:0 \ - pkg audit -f vuln.xml + pkg audit -f vuln.json } audit_out_of_range_body() { setup_packages # Vulnerability only affects versions < 1.0 - cat > vuln.xml << 'EOF' - - - - Old vulnerability - - - test - - 1.0 - - - - - CVE-2020-99999 - - - + cat > vuln.json << 'EOF' +[ + { + "affected": [ + { + "package": { + "ecosystem": "FreeBSD:ports", + "name": "test" + }, + "ranges": [ + { + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0" + } + ], + "type": "ECOSYSTEM" + } + ] + } + ], + "database_specific": { + "discovery": "2020-12-31T00:00:00Z", + "references": { + "cvename": [ + "CVE-2020-99999" + ] + }, + "vid": "old-vuln" + }, + "details": "Old vulnerability description\n", + "id": "FreeBSD-2020-0001", + "modified": "2020-12-31T00:00:00Z", + "published": "2020-12-31T00:00:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://cveawg.mitre.org/api/cve/CVE-2020-99999" + } + ], + "schema_version": "1.7.0", + "summary": "Old vulnerability" + } +] EOF # test-1.5 >= 1.0, so not affected atf_check \ -o match:"0 problem" \ -s exit:0 \ - pkg audit -f vuln.xml + pkg audit -f vuln.json } audit_quiet_body() { @@ -129,7 +184,7 @@ audit_quiet_body() { atf_check \ -o inline:"test-1.5\n" \ -s exit:1 \ - pkg audit -qf vuln.xml + pkg audit -qf vuln.json } audit_recursive_body() { @@ -153,7 +208,7 @@ EOF -o match:"test-1.5 is vulnerable" \ -o match:"rdep" \ -s exit:1 \ - pkg audit -rf vuln.xml + pkg audit -rf vuln.json } audit_raw_json_body() { @@ -165,7 +220,7 @@ audit_raw_json_body() { atf_check \ -o save:out.json \ -s exit:1 \ - pkg audit -f vuln.xml -Rjson + pkg audit -f vuln.json -Rjson # Must be valid JSON atf_check -o ignore -e empty python3 -m json.tool out.json @@ -194,7 +249,7 @@ audit_raw_ucl_body() { -o match:"version.*1.5" \ -o match:"CVE-2024-00001" \ -s exit:1 \ - pkg audit -f vuln.xml -R + pkg audit -f vuln.json -R } audit_pattern_body() { @@ -210,53 +265,110 @@ audit_pattern_body() { atf_check \ -o match:"1 problem" \ -s exit:1 \ - pkg audit -f vuln.xml test + pkg audit -f vuln.json test # Audit only the safe package by name atf_check \ -o match:"0 problem" \ -s exit:0 \ - pkg audit -f vuln.xml safe + pkg audit -f vuln.json safe } audit_multiple_vulns_body() { setup_packages # Two vulnerabilities affecting the same package - cat > vuln.xml << 'EOF' - - - - First vulnerability - - - test - - 1.0 - 2.0 - - - - - CVE-2024-00001 - - - - Second vulnerability - - - test - - 1.0 - 1.5 - - - - - CVE-2024-00002 - - - + cat > vuln.json << 'EOF' +[ + { + "affected": [ + { + "package": { + "ecosystem": "FreeBSD:ports", + "name": "test" + }, + "ranges": [ + { + "events": [ + { + "introduced": "1.0" + }, + { + "last_affected": "1.5" + } + ], + "type": "ECOSYSTEM" + } + ] + } + ], + "database_specific": { + "discovery": "2024-01-02T00:00:00Z", + "references": { + "cvename": [ + "CVE-2024-00002" + ] + }, + "vid": "vuln-002" + }, + "details": "Second vulnerability description\n", + "id": "FreeBSD-2024-0003", + "modified": "2024-01-02T00:00:00Z", + "published": "2024-01-02T00:00:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://cveawg.mitre.org/api/cve/CVE-2024-00002" + } + ], + "schema_version": "1.7.0", + "summary": "Second vulnerability" + }, + { + "affected": [ + { + "package": { + "ecosystem": "FreeBSD:ports", + "name": "test" + }, + "ranges": [ + { + "events": [ + { + "introduced": "1.0" + }, + { + "fixed": "2.0" + } + ], + "type": "ECOSYSTEM" + } + ] + } + ], + "database_specific": { + "discovery": "2024-01-01T00:00:00Z", + "references": { + "cvename": [ + "CVE-2024-00001" + ] + }, + "vid": "vuln-001" + }, + "details": "First vulnerability description\n", + "id": "FreeBSD-2024-0004", + "modified": "2024-01-01T00:00:00Z", + "published": "2024-01-01T00:00:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://cveawg.mitre.org/api/cve/CVE-2024-00001" + } + ], + "schema_version": "1.7.0", + "summary": "First vulnerability" + } +] EOF # Both vulnerabilities should be reported @@ -265,7 +377,7 @@ EOF -o match:"CVE-2024-00002" \ -o match:"2 problem" \ -s exit:1 \ - pkg audit -f vuln.xml + pkg audit -f vuln.json } audit_multiple_packages_body() { @@ -276,24 +388,53 @@ audit_multiple_packages_body() { atf_check -s exit:0 sh ${RESOURCEDIR}/test_subr.sh new_pkg "safe" "safe" "1.0" "/usr/local" atf_check -o ignore pkg register -M safe.ucl - cat > vuln.xml << 'EOF' - - - - Vulnerability in vuln package - - - vuln - - 2.0 - - - - - CVE-2024-99999 - - - + cat > vuln.json << 'EOF' +[ + { + "affected": [ + { + "package": { + "ecosystem": "FreeBSD:ports", + "name": "vuln" + }, + "ranges": [ + { + "events": [ + { + "introduced": "0" + }, + { + "fixed": "2.0" + } + ], + "type": "ECOSYSTEM" + } + ] + } + ], + "database_specific": { + "discovery": "2024-12-31T00:00:00Z", + "references": { + "cvename": [ + "CVE-2024-99999" + ] + }, + "vid": "vuln-pkg-001" + }, + "details": "Vulnerability in vuln package description\n", + "id": "FreeBSD-2024-0002", + "modified": "2024-12-31T00:00:00Z", + "published": "2024-12-31T00:00:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://cveawg.mitre.org/api/cve/CVE-2024-99999" + } + ], + "schema_version": "1.7.0", + "summary": "Vulnerability in vuln package" + } +] EOF # Only vuln package should be flagged @@ -301,13 +442,13 @@ EOF -o match:"vuln-1.0 is vulnerable" \ -o match:"1 problem.*1 package" \ -s exit:1 \ - pkg audit -f vuln.xml + pkg audit -f vuln.json # Quiet mode should only list the vulnerable one atf_check \ -o inline:"vuln-1.0\n" \ -s exit:1 \ - pkg audit -qf vuln.xml + pkg audit -qf vuln.json } audit_glob_name_body() { @@ -315,24 +456,53 @@ audit_glob_name_body() { atf_check -s exit:0 sh ${RESOURCEDIR}/test_subr.sh new_pkg "perl5-DBI" "perl5-DBI" "1.5" "/usr/local" atf_check -o ignore pkg register -M perl5-DBI.ucl - cat > vuln.xml << 'EOF' - - - - Vulnerability in perl DBI - - - perl5*-DBI - - 2.0 - - - - - CVE-2024-55555 - - - + cat > vuln.json << 'EOF' +[ + { + "affected": [ + { + "package": { + "ecosystem": "FreeBSD:ports", + "name": "perl5-DBI" + }, + "ranges": [ + { + "events": [ + { + "introduced": "0" + }, + { + "fixed": "2.0" + } + ], + "type": "ECOSYSTEM" + } + ] + } + ], + "database_specific": { + "discovery": "2024-06-15T00:00:00Z", + "references": { + "cvename": [ + "CVE-2024-55555" + ] + }, + "vid": "perl-vuln-001" + }, + "details": "Vulnerability in perl DBI description\n", + "id": "FreeBSD-2024-0001", + "modified": "2024-06-15T00:00:00Z", + "published": "2024-06-15T00:00:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://cveawg.mitre.org/api/cve/CVE-2024-55555" + } + ], + "schema_version": "1.7.0", + "summary": "Vulnerability in perl DBI" + } +] EOF # Glob pattern in vuln DB should match perl5-DBI @@ -340,7 +510,7 @@ EOF -o match:"perl5-DBI-1.5 is vulnerable" \ -o match:"CVE-2024-55555" \ -s exit:1 \ - pkg audit -f vuln.xml + pkg audit -f vuln.json } audit_no_db_body() { @@ -350,5 +520,5 @@ audit_no_db_body() { atf_check \ -e match:"does not exist" \ -s exit:1 \ - pkg audit -f /nonexistent/vuln.xml + pkg audit -f /nonexistent/vuln.json } diff --git a/tests/frontend/http.sh b/tests/frontend/http.sh index 9ca878f430..490075c8b6 100644 --- a/tests/frontend/http.sh +++ b/tests/frontend/http.sh @@ -108,23 +108,107 @@ simple_audit_body() { atf_require python3 "Requires python3 to run this test" - mkdir rootdir - cat > ${TMPDIR}/rootdir/meh < - - + mkdir -p ${TMPDIR}/rootdir + cat > ${TMPDIR}/rootdir/meh.json < Lorem ipsum dolor sit amet,\n>\n> consectetur adipiscing elit,\n>\n> sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", + "id": "FreeBSD-2025-0001", + "modified": "2025-11-12T00:00:00Z", + "published": "2025-11-11T00:00:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://www.securityfocus.com/bid/BBBBBBBB/info" + }, + { + "type": "ADVISORY", + "url": "https://www.cert.org/advisories/CA-XXXX-YY.html" + }, + { + "type": "ADVISORY", + "url": "https://www.kb.cert.org/vuls/id/CCCCCCCC" + }, + { + "type": "REPORT", + "url": "https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=SSSSSSSS" + }, + { + "type": "ADVISORY", + "url": "https://www.freebsd.org/security/advisories/FreeBSD-SA-XX:YY.bar.asc" + }, + { + "type": "ADVISORY", + "url": "https://api.osv.dev/v1/vulns/CVE-WWWW-WWWW" + }, + { + "type": "DISCUSSION", + "url": "https://www.freebsd.org/community/mailinglists/" + }, + { + "type": "WEB", + "url": "https://www.freebsd.org/about/" + } + ], + "schema_version": "1.7.0", + "summary": "bar -- some vulnerabilities" + } +] EOF - bzip2 ${TMPDIR}/rootdir/meh - httpd_startup ${TMPDIR}/rootdir cat > pkg.conf << EOF PKG_DBDIR=${TMPDIR} -VULNXML_SITE = "${url}/meh.bz2" +OSVF_SITE = "${url}/meh.json" EOF atf_check -o ignore pkg -C ./pkg.conf audit -F - + atf_check -o ignore pkg -C ./pkg.conf audit -F bar-1.1 } simple_audit_cleanup() { diff --git a/tests/frontend/install.sh b/tests/frontend/install.sh index 857b11bd8b..8c29a40a7f 100644 --- a/tests/frontend/install.sh +++ b/tests/frontend/install.sh @@ -633,26 +633,54 @@ local: { } EOF - # Create vuln.xml in PKG_DBDIR (current dir) - cat > vuln.xml << 'VULN' - - - - Test vulnerability - - - vuln - - 1.0 - 2.0 - - - - - CVE-2024-00001 - - - + # Create freebsd-osv.json in PKG_DBDIR (current dir) + cat > freebsd-osv.json << 'VULN' +[ + { + "affected": [ + { + "package": { + "ecosystem": "FreeBSD:ports", + "name": "vuln" + }, + "ranges": [ + { + "events": [ + { + "introduced": "1.0" + }, + { + "fixed": "2.0" + } + ], + "type": "ECOSYSTEM" + } + ] + } + ], + "database_specific": { + "discovery": "2024-01-01T00:00:00Z", + "references": { + "cvename": [ + "CVE-2024-00001" + ] + }, + "vid": "test-vuln-001" + }, + "details": "Example description\n", + "id": "FreeBSD-2024-0001", + "modified": "2024-01-01T00:00:00Z", + "published": "2024-01-01T00:00:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://cveawg.mitre.org/api/cve/CVE-2024-00001" + } + ], + "schema_version": "1.7.0", + "summary": "Test vulnerability in test package" + } +] VULN atf_check \ @@ -686,25 +714,53 @@ local: { EOF # Vulnerability only affects versions < 2.0 - cat > vuln.xml << 'VULN' - - - - Test vulnerability - - - safe - - 1.0 - 2.0 - - - - - CVE-2024-00001 - - - + cat > freebsd-osv.json << 'VULN' +[ + { + "affected": [ + { + "package": { + "ecosystem": "FreeBSD:ports", + "name": "safe" + }, + "ranges": [ + { + "events": [ + { + "introduced": "1.0" + }, + { + "fixed": "2.0" + } + ], + "type": "ECOSYSTEM" + } + ] + } + ], + "database_specific": { + "discovery": "2024-01-01T00:00:00Z", + "references": { + "cvename": [ + "CVE-2024-00001" + ] + }, + "vid": "test-vuln-001" + }, + "details": "Example description\n", + "id": "FreeBSD-2024-0001", + "modified": "2024-01-01T00:00:00Z", + "published": "2024-01-01T00:00:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://cveawg.mitre.org/api/cve/CVE-2024-00001" + } + ], + "schema_version": "1.7.0", + "summary": "Test vulnerability in test package" + } +] VULN atf_check \ @@ -724,7 +780,7 @@ VULN install_vulnerable_no_vuln_db_body() { - # Create a package and repo, but no vuln.xml + # Create a package and repo, but no freebsd-osv.json atf_check sh ${RESOURCEDIR}/test_subr.sh new_pkg "test" "test" "1.0" atf_check pkg create -M test.ucl atf_check -o ignore pkg repo . @@ -743,7 +799,7 @@ EOF pkg -o REPOS_DIR="${TMPDIR}/repoconf" -o PKG_CACHEDIR="${TMPDIR}" \ update - # No vuln.xml -> no vulnerability output, install proceeds normally + # No freebsd-osv.json -> no vulnerability output, install proceeds normally atf_check \ -o not-match:"\(vulnerable!\)" \ -o not-match:"WARNING.*vulnerabilities" \ diff --git a/tests/frontend/pkg.sh b/tests/frontend/pkg.sh index e28ae8586e..4f98451bd3 100755 --- a/tests/frontend/pkg.sh +++ b/tests/frontend/pkg.sh @@ -67,7 +67,8 @@ pkg_config_defaults_body() -o match:'^ *SYSLOG = true;$' \ -o match:'^ *ABI = "[a-zA-Z0-9]+:[a-z\.A-Z0-9]+:[a-zA-Z0-9_]+";$'\ -o match:'^ *DEVELOPER_MODE = false;$' \ - -o match:'^ *VULNXML_SITE = "https://vuxml.freebsd.org/freebsd/vuln.xml.xz";$' \ + -o match:'^ *VULNXML_SITE = "https://raw.githubusercontent.com/illuusio/freebsd-osv/refs/heads/main/db/freebsd-osv.json";$' \ + -o match:'^ *OSVF_SITE = "https://raw.githubusercontent.com/illuusio/freebsd-osv/refs/heads/main/db/freebsd-osv.json";$' \ -o match:'^ *FETCH_RETRY = 3;$' \ -o match:'^ *PKG_PLUGINS_DIR = ".*lib/pkg/";$' \ -o match:'^ *PKG_ENABLE_PLUGINS = true;$' \ diff --git a/tests/frontend/upgrade.sh b/tests/frontend/upgrade.sh index 179893d99d..aaa97b9e1e 100644 --- a/tests/frontend/upgrade.sh +++ b/tests/frontend/upgrade.sh @@ -1176,26 +1176,54 @@ local: { } EOF - # vuln.xml: test >=1.0 <2.0 is vulnerable - cat > vuln.xml << 'VULN' - - - - Test vulnerability - - - test - - 1.0 - 2.0 - - - - - CVE-2024-00001 - - - + # vuln.json: test >=1.0 <2.0 is vulnerable + cat > freebsd-osv.json << 'VULN' +[ + { + "affected": [ + { + "package": { + "ecosystem": "FreeBSD:ports", + "name": "test" + }, + "ranges": [ + { + "events": [ + { + "introduced": "1.0" + }, + { + "fixed": "2.0" + } + ], + "type": "ECOSYSTEM" + } + ] + } + ], + "database_specific": { + "discovery": "2024-01-01T00:00:00Z", + "references": { + "cvename": [ + "CVE-2024-00001" + ] + }, + "vid": "test-vuln-001" + }, + "details": "Example description\n", + "id": "FreeBSD-2024-0001", + "modified": "2024-01-01T00:00:00Z", + "published": "2024-01-01T00:00:00Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://cveawg.mitre.org/api/cve/CVE-2024-00001" + } + ], + "schema_version": "1.7.0", + "summary": "Test vulnerability in test package" + } +] VULN atf_check -o ignore \ diff --git a/tests/lib/FBSD-2025-05-28.json b/tests/lib/FBSD-2025-05-28.json index 4babc85bbb..9e21bee903 100644 --- a/tests/lib/FBSD-2025-05-28.json +++ b/tests/lib/FBSD-2025-05-28.json @@ -12,39 +12,39 @@ }, { "type": "ARTICLE", - "url": "https://www.freebsd.org/" + "url": "https://www.freebsd.org/about/" }, { "type": "DETECTION", - "url": "https://www.freebsd.org/" + "url": "https://docs.freebsd.org/en/" }, { "type": "DISCUSSION", - "url": "https://www.freebsd.org/" + "url": "https://docs.freebsd.org/en/books/handbook/basics/" }, { "type": "REPORT", - "url": "https://www.freebsd.org/" + "url": "https://wiki.freebsd.org/" }, { "type": "FIX", - "url": "https://www.freebsd.org/" + "url": "https://lists.freebsd.org/" }, { "type": "INTRODUCED", - "url": "https://www.freebsd.org/" + "url": "https://wiki.freebsd.org/IRC/Channels" }, { "type": "PACKAGE", - "url": "https://www.freebsd.org/" + "url": "https://docs.freebsd.org/en/books/" }, { "type": "EVIDENCE", - "url": "https://www.freebsd.org/" + "url": "hhttps://www.freebsd.org/releases/" }, { "type": "WEB", - "url": "https://www.freebsd.org/" + "url": "https://www.freebsd.org/releng/" } ], "affected": [ diff --git a/tests/lib/pkg_osvf.c b/tests/lib/pkg_osvf.c index 68cd4587e1..1fde6b1b44 100644 --- a/tests/lib/pkg_osvf.c +++ b/tests/lib/pkg_osvf.c @@ -12,7 +12,6 @@ #include #include - char *osvf_json_path = TESTING_TOP_DIR "/lib/FBSD-2025-05-28.json"; ATF_TC_WITHOUT_HEAD(osvfdetect); @@ -205,6 +204,19 @@ ATF_TC_BODY(osvfparse, tc) "osvf-test-package12", "osvf-test-package13" }; + char *refrence_str[] = + { + "https://www.freebsd.org/", + "https://www.freebsd.org/about/", + "https://docs.freebsd.org/en/", + "https://docs.freebsd.org/en/books/handbook/basics/", + "https://wiki.freebsd.org/", + "https://lists.freebsd.org/", + "https://wiki.freebsd.org/IRC/Channels", + "https://docs.freebsd.org/en/books/", + "hhttps://www.freebsd.org/releases/", + "https://www.freebsd.org/releng/" + }; int reference_types[] = { OSVF_REFERENCE_ADVISORY, @@ -240,11 +252,16 @@ ATF_TC_BODY(osvfparse, tc) references = oentry->references; + pos = 0; + otherpos = 0; + while(references) { - ATF_CHECK_STREQ(references->url, "https://www.freebsd.org/"); - ATF_CHECK_INTEQ(references->type, reference_types[pos++]); + ATF_CHECK_STREQ(references->url, refrence_str[otherpos]); + ATF_CHECK_INTEQ(references->type, reference_types[pos]); references = references->next; + otherpos++; + pos ++; } /* Check all versions across all packages (flat iteration) */