Speeding up make when getting information about pkgsrc packages

nettester

A little update to the whole thing (fully parallelized and quite optimized - caches bmake output, encodes option combinations as bitmasks, uses a client/server approach for feeding the workers, can use memory backed storage for most data, option testing is optional, actually bothers to correctly setup SQLite transactions and storage - compared to the last version):

#!/usr/pkg/bin/jimsh

set g_path_base /home/user/mnt
set g_path_ram /var/shm
set g_path_pkg $g_path_base/pkg
set g_path_src $g_path_base/pkgsrc
set g_path_make $g_path_pkg/bin/bmake
set g_path_log $g_path_base/log/db_build.log
set g_path_cache $g_path_base/cache
set g_path_db $g_path_base/package.db
set g_path_sock /tmp/dbb-sock
set g_blacklist [list audio/lame]

set g_cfg [dict create \
	threads 8 \
	debug 0 \
	update 0 \
	append 0 \
	blacklist 1 \
	package {} \
	cache 0 \
	test 1 \
	ram 0 \
]
set g_loop 1
set g_threads 0
set g_thread -1
set g_log {}
set g_log_buf {}
set g_db {}
set g_dbg {}
set g_sock {}
set g_pkg_list {}
set g_pkg_index 0
set g_opt_list {}
set g_finish 0
set g_stat 0
set g_stat_last 0
set g_stat_cnt 1

proc _dbg_cb { type fn line res cmd arglist } {
	global g_dbg
	if { $fn == {binary.tcl} || $cmd == {dbg_logend} || $cmd == {xtrace} } {
		return
	}
	if { [set l [llength $g_dbg]] > 20 } {
		set g_dbg [lrange $g_dbg 1 end]
	}
	if { $l > 0 } {
		set e [lindex $g_dbg end]
		lset e 3 $res
		lset g_dbg end $e
	}
	lappend g_dbg [list $type $fn $line {} $cmd $arglist]
}

proc dbg_log {} {
	global g_dbg
	set r {}
	foreach e $g_dbg {
		set l "## [lindex $e 0] [lindex $e 1]:[lindex $e 2]"
		lappend r "$l [lindex $e 4] [lindex $e 5] ==> \[[lindex $e 3]\]"
	}
	return [join $r \n]
}

proc dbg_logend {} {
	xtrace {}
}

proc str2hex { str } {
	binary scan $str H* h
	return $h
}

proc str2hash { str } {
	return [exec sha256 <<"$str"]
}

proc ver_cmp { v1 v2 } {
	if { $v1 == $v2 } {
		return -1
	}
	set v1 [split v1 .]
	set v2 [split v2 .]
	for { set i 0; set r -1 } { $i < [llength $v1] } { incr i } {
		if { $i == [llength $v2] } {
			return 1
		}
		if { [set n1 [lindex $v1 $i]] == [set n2 [lindex $v2 $i]] } {
			set r -1
			continue
		}
		set r [lindex [lsort -stride 2 -index 1 [list 0 $v1 1 $v2]] 0]
	}
	return $r
}

proc req_parse { req } {
	if { $req == {*any} } {
		return [list any {}]
	} elseif { [set t [string range $req 0 1]] == {>=} } {
		return [list $t [string range $req 2 end]]
	} elseif { [set t [string range $req 0 0]] == {=} } {
		return [list $t [string range $req 1 end]]
	}
	return [list exp $req]
}

proc req_bld { req } {
	switch -exact -- [lindex $req 0] {
		any { return *any }
		exp { return [lindex $req 1] }
	}
	return [join $req {}]
}

proc out_print { msg } {
	global g_log g_log_buf g_cfg
	if { [dict get $g_cfg threads] == 1 } {
		puts $msg
		$g_log puts $msg
		$g_log flush
		return
	}
	lappend g_log_buf $msg
}

proc out_flush {} {
	global g_log g_log_buf g_cfg
	if { [dict get $g_cfg threads] == 1 } return
	set s [join $g_log_buf \n]
	set g_log_buf {}
	$g_log lock -wait
	puts $s
	$g_log puts $s
	$g_log flush
	$g_log unlock
}

proc out_now { msg } {
	global g_log g_cfg
	if { [dict get $g_cfg threads] != 1 } {
		$g_log lock -wait
	}
	puts $msg
	$g_log puts $msg
	$g_log flush
	if { [dict get $g_cfg threads] != 1 } {
		$g_log unlock
	}
}

proc pkg_var { path name args } {
	global g_path_make g_path_src env
	set h [str2hash [join $args { }]]
	set env [dict create ENV {}]
	if { [catch {
		exec $g_path_make -C $g_path_src/$path \
			show-var VARNAME=$name {*}$args 2>/dev/null
	} v] } {
		set m __PROCESS_DIED__
		set m "$m [regsub {  *} [string map [list \n { }] $v] { }]"
		out_print "## $m"
		error $m
	}
	return $v
}

proc pkg_name { path } {
	return [regsub {^(.*)-[^-]*$} [pkg_var $path PKGNAME] {\1}]
}

proc pkg_ver { path } {
	return [regsub {^.*-([^-]*)$} [pkg_var $path PKGNAME] {\1}]
}

proc pkg_cat { path } {
	return [regsub {^([^/]*)/[^/]*$} $path {\1}]
}

proc pkg_com { path } {
	return [pkg_var $path COMMENT]
}

proc pkg_lic { path } {
	return [pkg_var $path LICENSE]
}

proc pkg_dsc { path } {
	global g_path_src
	if { ![file exists $g_path_src/$path/DESCR] } {
		return {}
	}
	set f [open $g_path_src/$path/DESCR r]
	set d [read $f]
	close $f
	return $d
}	

proc _pkg_dep_sanitize { deps } {
	set d {}
	foreach p [lsort [dict keys $deps]] {
		set l [dict get $deps $p]
		set v {}
		set e {}
		foreach r $l {
			set c [req_parse $r]
			if { $v == {} } {
				set v $c
				continue
			}
			switch -exact -- [lindex $c 0] {
				any {}
				exp { lappend e [lindex $c 1] }
				>= { switch -exact -- [lindex $v 0] {
					any { set v $c }
					>= {
						if {	[ver_cmp [lindex $c 1] \
							[lindex $v 1]] == 0 \
						} {
							set v $c
						}
					}
					= {
						if {	[ver_cmp [lindex $c 1] \
							[lindex $v 1]] == 0 \
						} {
							set v [list *imp {}]
							set e {}
							break
						}
					}
				} }
				= { switch -exact -- [lindex $v 0] {
					any {}
					>= {
						if {	[ver_cmp [lindex $c 1] \
							[lindex $v 1]] == 0 \
						} {
							set v [list *imp {}]
						}
					}
					= {
						set v [list *imp {}]
						set e {}
						break
					}
				} }
			}
		}
		if { $e != {} } {
			lappend d \
				"$p [join [list [req_bld $v] {*}[lsort $e] { }]]"
		} else {
			lappend d "$p [req_bld $v]"
		}
	}
	return $d
}

proc _pkg_dep { path type { opts {} } } {
	global g_db
	if { $opts != {} } {
		set opts [list "PKG_OPTIONS.[pkg_name $path]=$opts"]
	}
	set p [split [pkg_var $path $type {*}$opts] { }]
	set r {}
	foreach l $p {
		if { $l == {} } continue
		set l [split $l :]
		set n [regsub {^.*/([^/]*/[^/]*)$} [lindex $l 1] {\1}]
		set n [pkg_name $n]
		set v [regsub "^[string map [list + \\+] $n]" [lindex $l 0] {}]
		if { [string index $v 0] == {-} } {
			set v [string range $v 1 end]
		} elseif { $v == {} } {
			set v *any
		}
		if { [dict exists $r $n] } {
			dict set r $n [list {*}[dict get $r $n] $v]
		} else {
			dict set r $n $v
		}
	}
	return [_pkg_dep_sanitize $r]
}

proc pkg_dep { path { opt {} } } {
	return [_pkg_dep $path DEPENDS $opt]
}

proc pkg_deptool { path { opt {} } } {
	return [_pkg_dep $path TOOL_DEPENDS $opt]
}

proc pkg_depbld { path { opt {} } } {
	return [_pkg_dep $path BUILD_DEPENDS $opt]
}

proc pkg_optall { path } {
	return [join [lsort -unique \
		[split [pkg_var $path PKG_SUPPORTED_OPTIONS] { }]] { }]
}

proc pkg_optdef { path } {
	return [pkg_var $path PKG_OPTIONS]
}

proc pkg_optnorm { path opts } {
	if { $path == {-} } {
		return [lsort -unique $opts]
	}
	set a [pkg_optall $path]
	set d [pkg_optdef $path]
	set r {}
	foreach o $opts {
		if { [lsearch $d $o] != -1 } continue
		set n [regsub {^-} $o {}]
		if { [lsearch $d $n] != -1 } continue
		lappend r $o
	}
	return [lsort -unique $r]
}

proc pkg_optlst { opts def } {
	set r {}
	foreach o $opts {
		if { [lsearch $def $o] != -1 } {
			set o -$o
		}
		lappend r $o
	}
	return [lsort $r]
}

proc pkg_opthash { path opts } {
	switch -- [set r [str2hash [pkg_optnorm $path $opts]]] {
		12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126 {
			set r *def
		}
	}
	return $r
}

proc opt_test { path { opts {} } } {
	global g_path_src g_path_make g_cfg env
	if { ![dict get $g_cfg test] } {
		return {}
	}
	if { $opts != {} } {
		set opts [list "PKG_OPTIONS.[pkg_name $path]=[join $opts { }]"]
	}
	set env [dict create ENV {}]
	if { [catch {
		exec $g_path_make -C $g_path_src/$path \
			can-be-built-here {*}$opts 2>/dev/null
	} c] } {
		set m __PROCESS_DIED__
		set m "$m [regsub {  *} [string map [list \n { }] $c] { }]"
		out_print "## $m"
		return $m
	}
	set c [split $c \n]
	if { [lindex $c 0] == {yes} } {
		return {}
	}
	set s 0
	set r {}
	foreach l $c {
		if { $s } {
			append r " $l"
		} elseif { [string first PKG_FAIL_REASON $l] != -1 } {
			set s 1
		}
	}
	if { !$s } {
		return {unknown reason}
	}
	set r [string trim $r]
	set r [string map [list {	} { }] $r]
	return [regsub -all {  *} $r { }]
}

proc pkg_scanopt { path opts start end } {
	global g_db
	set count [expr $end - $start]
	out_print "-- validating $count variants for '$path' ..."
	set var 0
	set val {}
	for { set i $start } { $i < $end } { incr i } {
		for { set j 0; set opt {} } { $j < [llength $opts] } { incr j } {
			if { $i & ( 1 << $j ) } {
				lappend opt [lindex $opts $j]
			}
		}
		out_print " . trying options '$opt' for '$path' ..."
		if { [set reason [opt_test $path $opt]] != {} } {
			out_print " ! options '$opt' failed for '$path' = $reason"
			set dep {}
			set depbld {}
			set deptool {}
		} else {
			out_print " > validated options '$opt' for '$path'"
			set dep [join [lsort [pkg_dep $path $opt]] :]
			set depbld [join [lsort [pkg_depbld $path $opt]] :]
			set deptool [join [lsort [pkg_deptool $path $opt]] :]
			out_print " ~ dep = $dep | $depbld | $deptool"
			incr var
		}
		set v "( '$path', $i, '$dep', '$depbld', '$deptool',"
		lappend val "$v X'[str2hex $reason]' )"
	}
	$g_db query {begin immediate}
	$g_db query "insert into dep values [join $val ,]"
	set q "update pkg set parsed = parsed + $count"
	$g_db query "$q where path='$path'"
	$g_db query commit
}

proc pkg_scan { path } {
	global g_db g_cfg g_blacklist g_sock g_path_sock g_thread
	out_print "-- scanning '$path' ..."
	if {	[dict get $g_cfg blacklist] &&
		[lsearch $g_blacklist $path] != -1
	} {
		out_print " `package blacklisted"
		return
	}
	if { [dict get $g_cfg append] } {
		$g_db query {begin deferred}
		set r [$g_db query "select * from pkg where path='$path'"]
		$g_db query commit
		if { [llength $r] } {
			set r [lindex $r 0]
			set p [dict get $r parsed]
			if { [dict get $r count] == $p } {
				out_print " & append: path '$path' exists"
				return
			}
			set l [dict get $r optlst]
			set m " | info = $path | [dict get $r name] |"
			set m "$m [dict get $r version] | [dict get $r comment]"
			set m "$m | [dict get $r license] | [dict get $r optdef]"
			out_print "$m | $l | [llength $l] | [dict get $r comment]"
			$g_sock sendto [list $g_thread push \
				$path [dict get $r optlst] $p] $g_path_sock
			return
		}
	}
	set ver [pkg_ver $path]
	if { [dict get $g_cfg update] } {
		$g_db query {begin deferred}
		set r [$g_db query "select version from pkg where path='$path'"]
		$g_db query commit
		if { [llength $r] } {
			if { [set v [dict get [lindex $r 0] version]] == $ver } {
				set m " & update: path '$path' exists with known"
				out_print "$m version '$ver'"
				return
			}
			set m " & update: updated version for path '$path' ("
			out_print "$m '$v' > '$ver' )"
			$g_db query {begin immediate}
			$g_db query "delete from pkg where path='$path'"
			$g_db query "delete from dep where path='$path'"
			$g_db query commit
		}
	}
	set name [pkg_name $path]
	set lic [pkg_lic $path]
	set com [pkg_com $path]
	set dsc [pkg_dsc $path]
	set optsup [pkg_optall $path]
	set optdef [pkg_optdef $path]
	set optlst [pkg_optlst $optsup $optdef]
	set optcnt [expr 1 << [llength $optlst]]
	set m " | info = $path | $name | $ver | $com | $lic |"
	out_print "$m $optdef | $optlst | $optcnt | $com"
	$g_db query {begin immediate}
	set q "insert into pkg values ( '$path', '$name', '$ver', $optcnt, 0, "
	append q "0, '$lic', '$optdef', '$optlst', X'[str2hex $com]',"
	$g_db query "$q X'[str2hex $dsc]' )"
	$g_db query commit
	$g_sock sendto [list $g_thread push $path $optlst 0] $g_path_sock
}

proc pkg_thread_msg {} {
	global g_sock g_thread g_loop g_path_sock g_cfg
	set tid [expr $g_thread + 1]
	if { [catch {$g_sock recvfrom 1024 a} m] } {
		out_print "++ reading message for thread $tid failed"
		out_flush
		exit 1
	}
	switch -exact -- [lindex $m 0] {
		pkg {
			set path [lindex $m 1]
			if { [dict get $g_cfg threads] > 1 } {
				out_now "?? thread $tid ([pid]): scan '$path'"
				out_print "== thread $tid ..."
			}
			if { [dict get $g_cfg debug] == 1 } { xtrace _dbg_cb }
			pkg_scan $path
			if { [dict get $g_cfg debug] == 1 } { xtrace {} }
			out_flush
			$g_sock sendto [list $g_thread pull] $g_path_sock
		}
		opt {
			if { [dict get $g_cfg debug] == 1 } { xtrace _dbg_cb }
			foreach l [lindex $m 1] {
				lassign $l p o s e
				if { [dict get $g_cfg threads] > 1 } {
					set m "?? thread $tid ([pid]): test "
					append m "[expr $e - $s] variants "
					append m "($s to $e of "
					append m "[expr 1 << [llength $o]])"
					out_now "$m for '$p'"
					out_print "== thread $tid ..."
				}
				pkg_scanopt $p $o $s $e
				out_flush
			}
			$g_sock sendto [list $g_thread pull] $g_path_sock
			if { [dict get $g_cfg debug] == 1 } { xtrace {} }
		}
		wait {
			sleep 60
			$g_sock sendto [list $g_thread pull] $g_path_sock
		}
		exit { set g_loop 0 }
		default {
			out_print \
				"++ unknown message type '[lindex $m 0] for thread $tid"
			out_flush
			exit 1
		}
	}
}

proc pkg_thread { idx } {
	global g_cfg g_db g_path_db g_sock g_path_sock g_thread
	sleep [expr 2 + ( $idx > 0 ) * 15]
	set g_thread $idx
	set tid [expr $idx + 1]
	set g_db [sqlite3.open $g_path_db]
	$g_db query {PRAGMA busy_timeout = 99999}
	$g_db query {PRAGMA journal_mode = WAL}
	$g_sock close -nodelete
	set g_sock [socket unix.dgram.server $g_path_sock.$idx]
	$g_sock readable {
		if { [catch {
			pkg_thread_msg
		} msg opts] } {
			dbg_logend
			global g_loop g_thread
			out_print "## thread [expr $g_thread + 1] exited"
			if { [dict exists $opts -errorcode] } {
				out_print "## $msg"
				out_print {##}
				set s [dict get $opts -errorinfo]
				for { set i 0 } { $i < [llength $s] } { incr i 3 } {
					if { [set p [lindex $s $i]] == {} } {
						set p {$GLOBAL$}
					}
					set f [lindex $s [expr $i + 1]]
					set l [lindex $s [expr $i + 2]]
					out_print "## $f:$l scope: $p"
				}
				if { [dict get $g_cfg debug] == 1 } {
					out_print "##\n[dbg_log]"
				}
			}
			out_flush
			set g_loop 0
		}
	}
	$g_sock sendto [list $idx pull] $g_path_sock
	vwait g_loop
	out_print "__ thread $tid finished"
	out_flush
}

proc usage {} {
	global argv0
	puts stderr "Usage: $argv0 ..."
	puts stderr {	-h		: display this help screen		| opt |}
	puts stderr {	-t [VALUE]	: number of threads in multi mode	| opt | 8}
	puts stderr {	-u		: update database (conflicts with -a)	| opt | false}
	puts stderr {	-a		: append database (conflicts with -u)	| opt | false}
	puts stderr {	-b		: disable blacklist			| opt | false}
	puts stderr {	-c		: keep existing make cache		| opt | false}
	puts stderr {	-p [VALUE]	: specify package to scan		| opt | unset}
	puts stderr {	-o		: do not test options			| opt | false}
	puts stderr {	-r		: use ram backed store			| opt | false}
	puts stderr {	-d		: enable debug output			| opt | false}
	puts stderr {}
	exit 111
}

proc init { argv  } {
	global g_path_db g_db g_path_log g_log g_cfg g_path_sock g_sock \
		g_path_base g_path_cache g_path_pkg g_path_src g_path_ram \
		g_path_make

set optv [dict create -t threads -p package]
for { set i 0; set c [llength $argv] } { $i < $c } { incr i } {
	set o [lindex $argv $i]
	if { [dict exists $optv $o] } {
		 if { $i == [expr $c - 1] } usage
		dict set g_cfg [dict get $optv $o] \
			[lindex $argv [expr $i + 1]]
		incr i
		continue
	}
	switch -exact -- $o {
		-o { dict set g_cfg test 0 }
		-d { dict set g_cfg debug 1 }
		-c { dict set g_cfg cache 1 }
		-u {
			if { [dict get $g_cfg append] } usage
			dict set g_cfg update 1
		}
		-a {
			if { [dict get $g_cfg update] } usage
			dict set g_cfg append 1
		}
		-b { dict set g_cfg blacklist 0 }
		-r { dict set g_cfg ram 1 }
		default usage
	}
}

if { [dict get $g_cfg ram] } {
	set g_path_db $g_path_ram/package.db
	set g_path_cache $g_path_ram/cache
	exec rm -rf $g_path_ram/pkg
	exec mkdir $g_path_ram/pkg
	exec cp -R $g_path_pkg/. $g_path_ram/pkg
	set g_path_pkg $g_path_ram/pkg
	exec rm -rf $g_path_ram/pkgsrc
	exec mkdir $g_path_ram/pkgsrc
	exec cp -R $g_path_src/. $g_path_ram/pkgsrc
	set g_path_src $g_path_ram/pkgsrc
	set g_path_make $g_path_pkg/bin/bmake
}

file delete -f $g_path_log $g_path_sock \
	{*}[glob -nocomplain $g_path_sock.*]
if { ![dict get $g_cfg update] && ![dict get $g_cfg append] } {
	file delete -f $g_path_db
}

set g_log [open $g_path_log w]

set g_db [sqlite3.open $g_path_db]
$g_db query {pragma busy_timeout=99999}
$g_db query {pragma journal_mode=WAL}
if { ![dict get $g_cfg update] && ![dict get $g_cfg append] } {
	$g_db query {begin immediate}
	$g_db query {
		create table pkg (
			path varchar(64) default '' not null,
	    		name varchar(32) default '' not null,
	    		version varchar(32) default '' not null,
	    		count int(4) default 0 not null,
	    		parsed int(4) default 0 not null,
	    		variants int(4) default 0 not null,
	    		license varchar(64) default '' not null,
	    		optdef varchar(1024) default '' not null,
	    		optlst varchar(1024) default '' not null,
	    		comment varchar(1024) default '' not null,
	    		descr varchar(2048) default '' not null,
	    		primary key( path )
	    	) without rowid
	}
	$g_db query {
		create table dep (
			path text default '' not null,
			opt int(4) default -1 not null,
			deppkg text default '' not null,
			depbuild text default '' not null,
			deptool text default '' not null,
			error varchar(256) default '' not null,
			primary key( path, opt )
		) without rowid
	}
	$g_db query commit
}

set g_sock [socket unix.dgram.server $g_path_sock]
$g_sock sockopt rcvbuf [expr [dict get $g_cfg threads] * 1024]
$g_sock readable msg

if { ![file exists $g_path_pkg/bin/bmake.real] } {
	file rename $g_path_pkg/bin/bmake \
		$g_path_pkg/bin/bmake.real
}
if { ![dict get $g_cfg cache] } {
	exec rm -rf $g_path_cache
}
set f [open $g_path_pkg/bin/bmake w]
$f puts "#!/bin/sh\n\nPATH_CACHE='$g_path_cache'\n"
$f puts {
	P="$( for A in "$@"; do echo "$A"; done | sha256 | sed \
		's|^\(...\)\(...\)\(...\)\(...\)\(.*\)$|\1/\2/\3/\4/\5|' )"
	P="$PATH_CACHE/$P"
	if [ -f "$P/exp" ]; then
		E="$( find "$P" -mindepth 1 -maxdepth 1 -name exp \
			-mmin +15 )"
		[ -z "$E" ] || find "$P" -mindepth 6 -type f -name use \
			-mmin +15 | sed 's|/use$||' | xargs rm -rf
	else
		touch "$P/exp"
	fi
	if	[ ! -f "$P/out" ] || \
		[ ! -f "$P/err" ] || \
		[ ! -f "$P/res" ]; \
	then
		mkdir -p "$P"
		#echo "$*" > "$P/cmd"
		"$0.real" "$@" MAKE="$0" > "$P/out" 2> "$P/err"
		echo $? > "$P/res"
	fi
	cat "$P/out"
	cat "$P/err" >&2
	touch "$P/use"
	exit "$( cat "$P/res" )"
}
$f close
exec chmod +x $g_path_pkg/bin/bmake
}

proc cleanup {} {
	global g_log g_sock
	$g_sock close
	$g_log close
	puts {__ finished}
}

proc frame {} {
	global g_threads g_loop g_stat g_stat_last g_stat_cnt g_db
	if { [lindex [wait -nohang -1] 0] != {NONE} } {
		incr g_threads -1
	}
	if { $g_stat == 60 } {
		set r [$g_db query {select count(*) from dep}]
		set r [dict get [lindex $r 0] count(*)]
		out_now "++ $r deps ([expr $r - $g_stat_last]/m - [expr $r / $g_stat_cnt]/m avg)"
		set g_stat 0
		set g_stat_last $r
		incr g_stat_cnt
	} else {
		incr g_stat
	}
	if { !$g_threads } {
		set g_loop 0
	} else {
		after 1000 frame
	}
}

proc msg {} {
	global g_pkg_list g_pkg_index g_opt_list g_sock g_finish g_cfg
	if { [catch {$g_sock recvfrom [expr 1024 * 1024] a} m] } return
	set t [lindex $m 0]
	switch -exact -- [lindex $m 1] {
		pull {
			if { [llength $g_opt_list] } {
				for 	{ set v {}; set c 100 } \
					{ $c > 0 && [llength $g_opt_list] } \
					{} \
				{
					lassign [lindex $g_opt_list 0] p o s e
					if { [set l [expr $e - $s]] < $c } {
						set r [list $p $o $s $e]
						set g_opt_list [lrange \
							$g_opt_list 1 end]
						set c [expr $c - $l]
					} else {
						set r [list $p $o $s \
							[expr $s + $c]]
						lset g_opt_list 0 [list $p $o \
							[expr $s + $c] $e]
						set c 0
					}
					lappend v $r
				}
				set r [list opt $v]
			} elseif { $g_pkg_index >= [llength $g_pkg_list] } {
				if { [lsearch $g_finish $t] == -1 } {
					lappend g_finish $t
				}
				if {	[llength $g_finish] == \
					[dict get $g_cfg threads] \
				} {
					set r exit
				} else {
					set r wait
				}
			} else {
				set r [list \
					pkg [lindex $g_pkg_list $g_pkg_index]]
				incr g_pkg_index
			}
    			$g_sock sendto $r $a
    		}
    		push {
    			set p [lindex $m 2]
    			set o [lindex $m 3]
    			set s [lindex $m 4]
    			lappend g_opt_list [list $p $o $s [expr 1 << [llength $o]]]
    		}
    		default {
    			puts "## unknown thread message '[lindex $m 0]'"
    		}
    	}
}

init $argv
if { [set g_pkg_list [dict get $g_cfg package]] == {} } {
	set g_pkg_list [split \
		[exec find $g_path_base/pkgsrc -type f \
		-name Makefile -mindepth 3 -maxdepth 3] \n]
	for { set i 0 } { $i < [llength $g_pkg_list] } { incr i } {
		lset g_pkg_list $i [regsub {^.*/([^/]*/[^/]*)/Makefile$} \
			[lindex $g_pkg_list $i] {\1}]
	}
}
set g_threads [dict get $g_cfg threads]
for { set i 0 } { $i < $g_threads } { incr i } {
	puts "## launching thread [expr $i + 1] ..."
    	if { ![os.fork] } {
		pkg_thread $i
		exit 0
	}
}
after 1000 frame
vwait g_loop
cleanup
exit 0

Sadly it's still way to slow. Even without testing the actual option combinations it would take a couple years just to parse musicpd (several billions of possible combinations) using 100 threads on a cloud instance with about 50 dedicated cores (around 1500 variations processed per minute).

I guess i'll make yet another attempt and rewrite in C using a full scale database installation as i feel at this points it's mostly the scripted nature and SQLite, which are holding it back from going faster. I'm not really convinced it can be made to go fast enough to make building a full DB feasable but i figure it's still worth to try it.

Edit: If anyone has some kind of ideas in regards to saving time while executing bmake i'm still very open to hear them as a ton of time is obviously spent on bmake executions.

rvp

nettester You should run multiple targets in each make invokation rather than only the single one (if I'm reading your TCL right...):

Compare:

$ time -p bmake can-be-built-here show-vars VARNAMES="PKGNAME PKGVERSION COMMENT PKG_SUPPORTED_OPTIONS PKG_OPTIONS"
yes
ImageMagick-7.1.1.29
7.1.1.29
Package for display and non-interactive manipulation of images
djvu doc fontconfig fpx ghostscript heif jp2 liblqr openexr tiff wmf x11 imagemagick-policy-limited imagemagick-policy-open imagemagick-policy-secure imagemagick-policy-websafe
doc fontconfig ghostscript heif imagemagick-policy-open jp2 liblqr tiff x11
real 1.20
user 1.09
sys 0.15
$

versus:

$ time -p sh -c 'bmake can-be-built-here; bmake show-var VARNAME=PKGNAME; bmake show-var VARNAME=PKGVERSION; bmake show-var VARNAME=COMMENT; bmake show-var VARNAME=PKG_SUPPORTED_OPTIONS; bmake show-var VARNAME=PKG_OPTIONS'
yes
ImageMagick-7.1.1.29
7.1.1.29
Package for display and non-interactive manipulation of images
djvu doc fontconfig fpx ghostscript heif jp2 liblqr openexr tiff wmf x11 imagemagick-policy-limited imagemagick-policy-open imagemagick-policy-secure imagemagick-policy-websafe
doc fontconfig ghostscript heif imagemagick-policy-open jp2 liblqr tiff x11
real 7.07
user 6.29
sys 1.01
$

Should work since the results are

a) printed on a single line
b) in the order of the given targets, and
c) with empty lines for targets which result in no output (or botched targets).

nettester Even without testing the actual option combinations it would take a couple years just to parse musicpd (several billions of possible combinations)

Didn't quite get this: how billion combinations?

nettester

rvp Wow, that's huge! Thanks! You've been reading the code perfectly correct (by the way, sorry for using of what is probably one of the least common scripting languages available - i know all the cool kids use Perl and Python but i've never been any good at those 😉), it does a lot of single bmake invocations pretty much in straight succession (i wouldn't even have thought that it might be possible to stuff VARNAME with multiple names). This could be a huge improvement as i guess a lot of the time inside bmake is spent on setting up the environment. I'll try to implement this as soon as i find some free time.

It also seems that i might have found a way to get my hands onto more processing power. Given that Hetzner just moved it's dedicated servers to hourly billing too it seems to be possible to just get a whole 32 core / 64 threads EPYC at about half of what i'm paying for the cloud instance and i wouldn't be surprised if what i was getting from the cloud would be pretty much just 48 threads of exactly that CPU, so in theory that would come down to 1/3 more processing power + whatever is lost on the virtualization and possible contention due to noisy neighbors. Combined with cutting down on calls to bmake this could come down to a seriously big speed improvement.

I'm getting a little bit excited. This might finally approach realistic territory 😃

rvp Didn't quite get this: how billion combinations?

Well, musicpd has about 30+ (i'm not 100% sure anymore but something in that ballpark) package options and all of those can either be set or not set. Basically one can view them as bits, so 30 options result in a very, very big number of possible configurations. Luckily having this amount of options is not overly common among packages but there's still a couple, which are just extremely configurable.

rvp

nettester (by the way, sorry for using of what is probably one of the least common scripting languages available

'S OK. I barely know TCL, except expect (which is an entirely different beast).

nettester Well, musicpd has about 30+ (i'm not 100% sure anymore but something in that ballpark) package options and all of those can either be set or not set

Hmm. OK, but, do all those combinations make sense (or even compilable?). I would've though only what's currently set for the package was worth considering.

nettester

rvp Hmm. OK, but, do all those combinations make sense (or even compilable?). I would've though only what's currently set for the package was worth considering.

Sadly most of them do. A good example is also ffmpeg. There's just a ton of formats/codecs that can either be there or not. Still a couple options now and then are actually parts of groups where setting multiple or none doesn't make sense. From my impression avoiding those doesn't make a lot of difference though (if a package has like 5 options with 3 being a group the difference is really negligible), so i haven't really bothered to check if there some obvious way to get the group configurations yet.

I guess if there's a way building a list of bitmasks for known invalid configurations wouldn't be too hard. The question is just if the added complexity is likely worth it for whatever gain those achieve in the end. Bad combinations will obviously already fail during can-be-build-here anyways.

rvp

nettester make show-options looks useful for pruning option combinations. Trying it on ffmpeg6 yields:

$ cd /tmp/pkgsrc/multimedia/ffmpeg6
$ bmake show-options
Any of the following general options may be selected:
        aom      Enable AV1 support using libaom.
        ass      Enable subtitle renderer for the ASS/SSA format
        bluray   Enable libbluray support.
        doc      Include the documentation.
        fdk-aac  Enable Fraunhofer FDK AAC codec support.
        fontconfig       Enable FONTCONFIG (fonts/fontconfig) support.
        freetype         Enable freetype support.
        jack     Enable support for the JACK audio server.
        lame     Use the lame encoder to create MP3 files.
        libvpx   Enable libvpx support.
        libwebp  Enable libwebp support.
        opencore-amr     Enable OpenCORE Adaptive Multi-Rate speech codecs.
        opus     Enable opus support.
        pulseaudio       Enable support for the PulseAudio sound server.
        rav1e    Enable AV1 encoding support using rav1e.
        rpi      Enable Raspberry Pi support.
        rtmp     Enable rtmp:// support using rtmpdump.
        speex    Enable speex support.
        tesseract        Enable OCR support using Tesseract.
        theora   Use the theora codecs.
        vaapi    Enable support for VAAPI.
        vdpau    Enable support for VDPAU.
        vorbis   Enable Ogg Vorbis support.
        x11      Enable X11 support.
        x264     Enable x264 support.
        x265     Enable x265 support.
        xvid     Enable usage of XVid codecs.
At most one of the following ssl options may be selected:
        gnutls   Enable GNU TLS support.
        mbedtls  Enable mbed TLS support.
        openssl  Enable OpenSSL support.

These options are enabled by default:
        aom ass bluray fontconfig freetype gnutls
        lame libvpx libwebp opus speex theora vaapi
        vdpau vorbis x11 x264 x265 xvid

These options are currently enabled:
        aom ass bluray fontconfig freetype gnutls
        lame libvpx libwebp opus speex theora vaapi
        vdpau vorbis x11 x264 x265 xvid

You can select which build options to use by setting PKG_DEFAULT_OPTIONS
or PKG_OPTIONS.ffmpeg6.
$

Here, we can take it as given that any combination of options in the 1st section will build--barring compile errors, etc.--because any and all of those libraries will also be built if so enabled.

The next section is self-explanatory.

nettester Bad combinations will obviously already fail during can-be-build-here anyways.

can-be-built-here only checks if the package can be built on the current OS and using the current compiler. I don't think it looks at package options... (Ref: make help topic=can-be-built-here)

nettester

rvp Yeah, it's just not exactly fun to parse this, even if in that case it would actually make a quite notable difference due the massive amount of other options. I'll look into it. I'm just a bit hesitant on this as i really want to move the whole thing to C and a real database (to rule out any choke points due to Jim/SQLite) and if there's anything that C isn't great at it's string parsing but i guess given it's actually about a couple hundred million combinations here just ignoring it would be kind of crazy.

I'm pretty sure that the assumption of legal options necessarily also building isn't generally true though but i guess it could still be a sane assumption to go by if actually checking them all proves infeasible. I have not really followed the build errors all that closely but i'd say those cases are rather rare.

can-be-build-here actually does bail due being unable to determine the setup for an illegal configuration but i guess it's probably rather a side effect of it trying to initialize the environment than some kind of intended functionality.

Edit: Well, i'm kind of thinking of switching to real database first as my suspicion is rather on the inserts causing locks (i might be expecting a bit to much but to me it just seems too slow even with build checking turned off) than the interpreter (as simple as it might be) choking on generating the query. The only problem with this is that i'll have to write client myself, which would still be way better than redoing the whole thing in C though. On the other hand if i still end up rewriting it's pretty much a waste of time, so hmm...

rvp

nettester I'm pretty sure that the assumption of legal options necessarily also building isn't generally true

A pkgsrc expert on the lists will be able to give definitive answers for this, but, I think that if doing a can-be-built-here in each of the directories corresponding to the package options returns yes, then you can be sure that any combination of those options will build. No need to check further for each of those combinations.

nettester can-be-build-here actually does bail due being unable to determine the setup for an illegal configuration

Can you show how you do this?

nettester

rvp Can you show how you do this?

Well, basically just running bmake with a target of can-be-build-here and the respective set of options. If i'm not misremembering this having two options out of a group set results in an error along the lines of "Only one of these can be set" while setting none would get "You need to select at least one of ...". Like i've said, i think it's a side effect of the environment setup (or specifically parsing package options) and not really intended functionality, which would result in an output of "no" to indicate the impossibility of such a build.

I'm a bit short on time right now otherwise i'd just run a test but i'll remember to get back to it once i'll sit down and work on this again.

You might have a point in general though. If can-be-build-here doesn't do more than checking the availability of dependencies and one assumes the there is no special cases for various combinations (like any option just possibly adding more dependencies) a check with every option set should pretty much be indicative of the maximum amount of dependencies being satisfied. I'm just not 100% sure yet if there couldn't be some kind of exception due to special scripting attached to an option or something like this.

rvp

nettester a check with every option set should pretty much be indicative of the maximum amount of dependencies being satisfied.

Yes... something like:

/home/bld/tmp/pkgsrc/audio/musicpd $ make PKG_SUGGESTED_OPTIONS='${PKG_SUPPORTED_OPTIONS}' can-be-built-here
yes
/home/bld/tmp/pkgsrc/audio/musicpd $ make PKG_SUGGESTED_OPTIONS='${PKG_SUPPORTED_OPTIONS} pipewire snapcast sid' can-be-built-here
no
This package has set PKG_FAIL_REASON:
[bsd.options.mk] The following selected options are not supported: pipewire sid snapcast.
/home/bld/tmp/pkgsrc/audio/musicpd $

(This was on NetBSD; on Ubuntu 23.10, the same bmake command went off into the weeds eating up all of my memory...)

However, there are a few wrinkles:
a)

$ uname -a
Linux CoreBook 6.5.0-28-generic #29-Ubuntu SMP PREEMPT_DYNAMIC Thu Mar 28 23:46:48 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux

$ cd /tmp/pkgsrc/graphics/ImageMagick6

$ bmake PKG_SUGGESTED_OPTIONS='${PKG_SUPPORTED_OPTIONS}' can-be-built-here
yes

$

even though I don't have any of the X11 devel stuff installed on this machine. (Presumably, because pkgsrc will download and compile X11 on its own to satisfy the dependency -- I passed --prefer-pkgsrc yes to ./bootstrap)

And,

b) it says yes even when all policies are selected (is this valid?):

$ bmake show-var VARNAME=PKG_SUPPORTED_OPTIONS
doc fontconfig ghostscript x11 jp2 djvu openexr wmf fpx imagemagick-policy-limited imagemagick-policy-open imagemagick-policy-secure imagemagick-policy-websafe

$ bmake show-var VARNAME=PKG_SUGGESTED_OPTIONS
doc fontconfig ghostscript x11 jp2 imagemagick-policy-open

$ bmake PKG_SUGGESTED_OPTIONS='${PKG_SUPPORTED_OPTIONS}' can-be-built-here
yes

$

I don't really have the time right now but, if you want to mix and match options from different groups, you need to rework the definition of the group.

nettester

pin I don't think it's that big of a deal. We are basically just scratching heads if can-be-build-here would indicate an illegal option combination at all.

Going further by the assumption that can-be-build-here could indicate the availability of dependencies across all possible combinations of options supplied, it might be advantageous to be able to set all of them (including illegal combinations) but i fear unless there's a simple way to make pkgsrc ignore groupings changing every single package script containing groups would be infeasible anyways.

nettester

rvp That's quite interesting. I'm a bit surprised by the use of PKG_SUGGESTED_OPTIONS though. Is there a difference to setting PKG_OPTIONS/PKG_OPTIONS.packagename?

In regards to ImageMagick6 setting all options seems to be valid as there are no groups defined here. net/wget would make a good testcase i think as it can use either openssl or gnutls and it's obviously only one or the other but not both.

rvp

nettester I'm a bit surprised by the use of PKG_SUGGESTED_OPTIONS though. Is there a difference to setting PKG_OPTIONS/PKG_OPTIONS.packagename?

Can't say; I'm no pkgsrc expert. I'm just overriding the stuff in the Makefiles.

nettester net/wget would make a good testcase i think as it can use either openssl or gnutls and it's obviously only one or the other but not both.

Yah, for those you'll have to a can-be-built-here for each option separately (after parsing this goop).

« Previous Page