#! /usr/bin/env tclsh
set domain "appbox.rkeene.org"
set packages $argv
lappend auto_path [file join [file dirname [info script]] .. lib]
package require tax
package require csv
package require sha1
package require md5
namespace eval ::cael::build {
variable pkgmap_fields [list name type owner group mode mtime size sha1 linkdest]
variable pkginfodir [file join [file dirname [info script]] .. pkginfo]
variable buildonlyvars [list destroot buildroot metadataroot extraroot]
variable have_tclx 0
variable default_scripts {
compile {
/bin/sh {
./configure "--prefix=${_APPBOX_PREFIX}" "--sysconfdir=${_APPBOX_SYSCONFDIR}" || exit 1
make || exit 1
exit 0
}
}
install {
/bin/sh {
make "DESTDIR=${destroot}" install || exit 1
exit 0
}
}
}
}
catch {
package require Tclx
set ::cael::build::have_tclx 1
}
if {!$::cael::build::have_tclx} {
proc recursive_glob {dirlist globlist} {
set result {}
set recurse {}
foreach dir $dirlist {
if ![file isdirectory $dir] {
error "\"$dir\" is not a directory"
}
foreach pattern $globlist {
set result [concat $result [glob -nocomplain -- [file join $dir $pattern]]]
}
foreach file [glob -nocomplain -tails -directory $dir * .*] {
set file [file join $dir $file]
if [file isdirectory $file] {
set fileTail [file tail $file]
if {$fileTail != "." && $fileTail != ".."} {
lappend recurse $file
}
}
}
}
if {[llength $recurse] != 0} {
set result [concat $result [recursive_glob $recurse $globlist]]
}
return $result
}
}
proc ::cael::build::_parse_pkgXML {tag isClosing isSelfClosing props_list body} {
if {[string match "!--*" $tag]} {
return
}
if {$isClosing} {
set lastOpened [lindex $::cael::build::_xml_stack end]
if {$tag != $lastOpened} {
return -code error "XML Parsing Error: Got close for tag \"${tag}\", but last opened was \"${lastOpened}\""
}
set fulltag [join $::cael::build::_xml_stack .]
switch -- $fulltag {
root.package.releases.release {
unset -nocomplain currversioninfo
# Convert array-like list into an array
foreach {var val} $::cael::build::_currversion {
switch -- $var {
"urls" - "sigs" - "alt" {
# These elements form lists
lappend currversioninfo($var) $val
}
default {
# All other elements form strings
set currversioninfo($var) $val
}
}
}
lappend ::cael::build::_parse_array(versions) [array get currversioninfo]
unset ::cael::build::_currversion
}
root.package.variants.variant {
set ::cael::build::_ignoreTags_until_variant 0
}
root.package.releases {
# If a default URL was specified, go back through every release and add an "urls" list if none are present
if {[info exists ::cael::build::_parse_array(urls)] && [info exists ::cael::build::_parse_array(versions)]} {
set newversions [list]
foreach version_ent_list $::cael::build::_parse_array(versions) {
array set tmpversinfo $version_ent_list
if {![info exists tmpversinfo(urls)]} {
set tmpversinfo(urls) $::cael::build::_parse_array(urls)
}
lappend newversions [array get tmpversinfo]
}
set ::cael::build::_parse_array(versions) $newversions
}
}
root.package {
foreach {scriptname} [lsort -dictionary [array name ::cael::build::_script_array]] {
set script_frags $::cael::build::_script_array($scriptname)
set interp $::cael::build::_script_info($scriptname)
set script_list [list]
foreach script_frag $script_frags {
foreach line [split $script_frag "\n"] {
lappend script_list $line
}
}
set script [join $script_list "\n"]
lappend ::cael::build::_parse_array(scripts) $scriptname [list $interp $script]
}
}
}
set ::cael::build::_xml_stack [lrange $::cael::build::_xml_stack 0 end-1]
return
}
regsub -all {\&ob;} $body \{ body
regsub -all {\&cb;} $body \} body
lappend ::cael::build::_xml_stack $tag
set fulltag [join $::cael::build::_xml_stack .]
array set props $props_list
if {$::cael::build::_ignoreTags_until_variant} {
set fulltag "__IGNORED__"
}
switch -- $fulltag {
__IGNORED__ {}
root.package {
set pkgname $props(name)
set pkgname [string trim $pkgname]
set ::cael::build::_parse_array(name) $pkgname
}
root.package.description {
set short_desc $props(short)
set long_desc $body
set short_desc [string trim $short_desc]
set long_desc [string trim $long_desc]
set ::cael::build::_parse_array(short_desc) $short_desc
set ::cael::build::_parse_array(long_desc) $long_desc
}
root.package.releases.release {
set ::cael::build::_currversion [list version $props(version)]
}
root.package.releases.release.alt {
lappend ::cael::build::_currversion alt $body
}
root.package.releases.release.source.url {
lappend ::cael::build::_currversion urls $body
}
root.package.releases.release.source.sig {
lappend ::cael::build::_currversion sigs $body
}
root.package.releases.release.sha1 {
lappend ::cael::build::_currversion sha1 [string trim $body]
}
root.package.releases.source.url {
lappend ::cael::build::_parse_array(urls) $body
}
root.package.releases.source.sig {
lappend ::cael::build::_parse_array(sigs) $body
}
root.package.variants.variant {
lappend ::cael::build::_parse_array(variants) $props(name)
if {$props(name) != $::cael::build::_variant} {
set ::cael::build::_ignoreTags_until_variant 1
} else {
set ::cael::build::_parse_array(variant) $props(name)
}
}
root.package.var - root.package.variants.variant.var {
lappend ::cael::build::_parse_array(env_vars) $props(name) $props(value)
}
root.package.script - root.package.variants.variant.script {
if {![info exists props(name)] && ![info exists props(names)]} {
set props(name) "immediate-$::cael::build::_script_num"
incr ::cael::build::_script_num
}
if {[info exists props(names)]} {
if {[info exists props(name)]} {
return -code error "May not specify both the \"name\" and \"names\" properties for a script"
}
set script_names [split $props(names) ","]
} else {
set script_names [list $props(name)]
}
foreach script_name $script_names {
set script_name [string trim $script_name]
if {[info exists props(interp)]} {
set ::cael::build::_script_info($script_name) $props(interp)
} else {
if {![info exists ::cael::build::_script_info($script_name)]} {
# The first reference to a script must include the interp
return -code error "Script \"$script_name\" has no interpreter (first reference must include an \"interp\" property to specify the interpreter)"
}
}
lappend ::cael::build::_script_array($script_name) $body
}
}
root.package.files.default {
set permslist [list]
foreach prop [list owner group filemode dirmode] {
lappend permslist $prop $props($prop)
}
set ::cael::build::_parse_array(default-file) $permslist
}
root.package.files.file {
set permslist [list]
foreach prop [list owner group mode] {
lappend permslist $prop $props($prop)
}
lappend ::cael::build::_parse_array(files) $props(name) $permslist
}
root.package.buildflags.static - root.package.variants.variant.buildflags.static {
lappend ::cael::build::_parse_array(buildflags) "static"
}
root - \
root.package.releases - \
root.package.releases.release.source - \
root.package.releases.source - \
root.package.variants - \
root.package.buildflags - \
root.package.variants.variant.buildflags - \
root.package.files {
# We don't handle these tags
}
default {
puts stderr "Unhandled Tag: <${fulltag} props=$props_list, body=[string trim $body]>"
}
}
if {$isSelfClosing} {
set ::cael::build::_xml_stack [lrange $::cael::build::_xml_stack 0 end-1]
}
return
}
proc ::cael::build::_parse_pkgXML_init {variant} {
set ::cael::build::_script_num 0
set ::cael::build::_xml_stack ""
set ::cael::build::_variant $variant
set ::cael::build::_ignoreTags_until_variant 0
unset -nocomplain ::cael::build::_parse_array ::cael::build::_currversion
set ::cael::build::_parse_array(variant) default
}
proc ::cael::build::_version_sort {a b} {
set foundval [lindex [lsort -dictionary [list $a $b]] 0]
if {$foundval == "$a"} {
return -1
}
return 1
}
proc ::cael::build::_parse_pkg {pkgname {version "latest"} {variant "default"}} {
set file [file join $::cael::build::pkginfodir "${pkgname}.xml"]
# Read in XML document
set fd [open $file r]
set document [read $fd]
close $fd
# Cleanup internal XML structures
::cael::build::_parse_pkgXML_init $variant
# Parse XML Document
::tax::parse ::cael::build::_parse_pkgXML $document root
# Populate version information
## Determine the latest version if it is requested
set versions [list]
array set allversinfo [list]
foreach versinfo_list $::cael::build::_parse_array(versions) {
unset -nocomplain versinfo
array set versinfo $versinfo_list
set item_version $versinfo(version)
lappend versions $item_version
set allversinfo($item_version) $versinfo_list
}
if {$version == "latest"} {
set version [lindex [lsort -decreasing -command ::cael::build::_version_sort $versions] 0]
}
set ::cael::build::_parse_array(versinfo) $allversinfo($version)
# Generate return value
set retval [array get ::cael::build::_parse_array]
# Cleanup
unset ::cael::build::_parse_array
return $retval
}
proc ::cael::build::_url_subst {url env version} {
# Create a safe interp to run the substitutions in
set interp [interp create -safe]
foreach {var val} $env {
$interp eval [list set $var $val]
}
set retval [$interp eval [list subst $url]]
interp delete $interp
return $retval
}
proc ::cael::build::script_gen_execute {shell args} {
# Determine the properties of this shell
switch -glob -- $shell {
"*/csh" {
lappend escapeMapping {'} {'"'"'}
set quoteOpen "'"
set quoteClose "'"
set abortSuffix " || exit 1"
set commentChar "#"
set execCmd ""
}
"*/tclsh" {
lappend escapeMapping "\\" "\\\\" "\"" "\\\"" "{" "\\\{" "}" "\\\}" "\$" "\\\$" "\[" "\\\[" "\]" "\\\]"
set quoteOpen "\""
set quoteClose "\""
set abortSuffix ""
set commentChar "#"
set execCmd "exec "
}
default {
# Bourne-like shells
lappend escapeMapping {'} {'"'"'}
set quoteOpen "'"
set quoteClose "'"
set abortSuffix " || exit 1"
set commentChar "#"
set execCmd ""
}
}
# Set a default value if we are unable to emit usable actions
set ret "echo \"Unable to determine how to \\\"[join $args " "]\\\"\"\nexit 1"
# Determine output
set cmd [lindex $args 0]
switch -- $cmd {
"set" {
set var [lindex $args 1]
set val [string map $escapeMapping [lindex $args 2]]
switch -glob -- $shell {
"*/csh" {
set ret "setenv $var='$val'"
}
"*/tclsh" {
set ret "set $var \"$val\""
}
default {
set ret "$var='$val'; export $var"
}
}
}
"mkdir" {
set dir [string map $escapeMapping [lindex $args 1]]
switch -glob -- $shell {
"*/tclsh" {
set ret "file mkdir -- \"$dir\""
}
default {
set ret "mkdir -p '$dir' || exit 1"
}
}
}
"mv" {
set src [string map $escapeMapping [lindex $args 1]]
set dst [string map $escapeMapping [lindex $args 2]]
switch -glob -- $shell {
"*/tclsh" {
set ret "file rename -- \"$src\" \"$dst\""
}
default {
set ret "mv '$src' '$dst' || exit 1"
}
}
}
"cd_one_dir" {
switch -glob -- $shell {
"*/csh" {
# XXX: TODO
}
"*/tclsh" {
# XXX: TODO
}
default {
set ret "_tmp_build_num_dirs=\"\`ls -1a | wc -l`\"\n"
append ret "if \[ \"x\${_tmp_build_num_dirs}\" = \"x3\" ]; then\n"
append ret "\tcd *\n"
append ret "fi\n"
append ret "unset _tmp_build_num_dirs\n"
}
}
}
"sha1check" {
set file [string map $escapeMapping [lindex $args 1]]
set sha1 [string map $escapeMapping [lindex $args 2]]
switch -glob -- $shell {
"*/csh" {
# XXX: TODO
}
"*/tclsh" {
# XXX: TODO
}
default {
set ret "_tmp_sha1sum=\"`sha1sum '${file}' | awk '{ print \$1 }'`\"\n"
append ret "if \[ \"x\${_tmp_sha1sum}\" != 'x${sha1}' \]; then\n"
append ret "\techo 'Checksum of \"${file}\" failed. Aborting.\'\n"
append ret "\texit 1\n"
append ret "fi\n"
append ret "unset _tmp_sha1sum\n"
}
}
}
"wget" {
set file [string map $escapeMapping [lindex $args 1]]
set urls [lindex $args 2]
switch -glob -- $shell {
"*/tclsh" {
# XXX: TODO
}
default {
set ret "wget -O '${file}' '[string map $escapeMapping [lindex $urls 0]]' || \\\n"
foreach url [lrange $urls 1 end] {
append ret "\twget -O '${file}' '[string map $escapeMapping $url]' || \\\n"
}
append ret "\texit 1\n"
}
}
}
"extract" {
set file [string map $escapeMapping [lindex $args 1]]
switch -glob -- $shell {
"*/tclsh" {
# XXX: TODO
}
default {
set ret "${execCmd}tar -xf ${quoteOpen}${file}${quoteClose} || \\\n"
append ret "\t${execCmd}xz -dc ${quoteOpen}${file}${quoteClose} | tar -xf - || \\\n"
append ret "\t${execCmd}unzip ${quoteOpen}${file}${quoteClose}${abortSuffix}"
}
}
}
"cd" {
set dir [string map $escapeMapping [lindex $args 1]]
set ret "cd ${quoteOpen}${dir}${quoteClose}${abortSuffix}"
}
}
return $ret
}
proc ::cael::build::gen_scripts {pkgname version variant workdir instrootdir versinfo_list pkginfo_list scripts_list} {
array set versinfo $versinfo_list
array set pkginfo $pkginfo_list
set ret [list]
set scriptdir [file join $workdir build-scripts]
set builddir [file join $workdir build]
set srcdir [file join $workdir src]
file mkdir $scriptdir
file mkdir $srcdir
foreach {script scriptval} $scripts_list {
# Sanitize script name to create script name
set script [file tail $script]
# Determine script file name
set file [file join $scriptdir "${script}"]
# Record script name and file for later use
lappend ret $script $file
set init 0
if {![file exists $file]} {
set init 1
}
set fd [open $file a+]
if {$init} {
set shell [lindex $scriptval 0]
# Create generic preamble
puts $fd "#!${shell}"
puts $fd ""
set export_vars [list]
foreach {var val} $pkginfo(env_vars) {
# Exclude build-related variables from
# non-build-related scripts
switch -- $script {
"compile" - "install" {
}
default {
if {[lsearch -exact $::cael::build::buildonlyvars $var] != -1} {
continue
}
}
}
puts $fd [script_gen_execute $shell set $var $val]
}
# Do script-specific preamble
switch -- $script {
"compile" - "install" {
puts $fd [script_gen_execute $shell mkdir $workdir]
puts $fd [script_gen_execute $shell cd $workdir]
puts $fd ""
}
}
# Do script-specific stuff
switch -- $script {
"compile" {
puts $fd [script_gen_execute $shell mkdir $srcdir]
puts $fd ""
set outfile [file join $srcdir "${pkgname}-$versinfo(version).src"]
# Download archive
set urls [list]
foreach url $versinfo(urls) {
lappend urls [_url_subst $url $pkginfo(env_vars) $versinfo(version)]
}
puts $fd [script_gen_execute $shell wget "${outfile}.new" $urls]
puts $fd [script_gen_execute $shell sha1check "${outfile}.new" $versinfo(sha1)]
puts $fd "mv '${outfile}.new' '${outfile}'"
puts $fd ""
# Extract archive
puts $fd [script_gen_execute $shell mkdir $builddir]
puts $fd [script_gen_execute $shell cd $builddir]
puts $fd [script_gen_execute $shell extract $outfile]
puts $fd ""
# Change to working directory within archive, if available
puts $fd [script_gen_execute $shell cd_one_dir]
}
"install" {
# Change to working directory within archive
puts $fd [script_gen_execute $shell cd $builddir]
puts $fd [script_gen_execute $shell cd_one_dir]
puts $fd ""
# Create the installation root directory
puts $fd [script_gen_execute $shell mkdir $instrootdir]
puts $fd ""
}
}
}
foreach part [lrange $scriptval 1 end] {
puts $fd $part
}
close $fd
}
return $ret
}
proc ::cael::build::gen_meta {pkgname pkgdomain version variant workdir instrootdir instmetadir versinfo_list pkginfo_list scripts_list} {
array set versinfo $versinfo_list
array set pkginfo $pkginfo_list
array set fileinfo [list]
if {[info exists pkginfo_list(files)]} {
array set fileinfo $pkginfo_list(files)
}
## Create meta directory
file mkdir $instmetadir
## Create Description file
set pkgdescfile [file join $instmetadir desc]
set fd [open $pkgdescfile w]
puts $fd "[join [split $pkginfo(short_desc) "\n"] " "]"
puts $fd "$pkginfo(long_desc)"
close $fd
## Create Package Map file
### Generate list of all files
set filelist [recursive_glob [list $instrootdir] [list *]]
### Determine how to modify the file name to be what is available
### to the system
set strip_file_names [string length $instrootdir]
### Create default file information if it was not provided
if {![info exists pkginfo(default-file)]} {
set pkginfo(default-file) [list]
}
### Create mapping
set pkgmapfile [file join $instmetadir map]
set fd [open $pkgmapfile w]
foreach file $filelist {
#### Ensure that we do not get data from another file if we fail
#### to create a field
unset -nocomplain destfile
#### Get current file information
file lstat $file srcfile
#### Set reasonable defaults
set destfile(owner) 0
set destfile(group) 0
set destfile(mode) [expr {$srcfile(mode) & 0777}]
#### Set derived parameters
set destfile(name) [string range $file $strip_file_names end]
set destfile(type) $srcfile(type)
set destfile(mtime) $srcfile(mtime)
if {$destfile(type) == "file"} {
set destfile(size) $srcfile(size)
set destfile(sha1) [sha1::sha1 -hex -file $file]
}
if {$destfile(type) == "link"} {
set linkdest [file link $file]
##### If the link points inside the package directory,
##### rewrite it
if {[string range $linkdest 0 [expr {$strip_file_names - 1}]] == [string range $file 0 [expr {$strip_file_names - 1}]]} {
set linkdest [string range $linkdest $strip_file_names end]
}
set destfile(linkdest) $linkdest
}
#### Set the default values
foreach {var val} $pkginfo(default-file) {
switch -- $var {
"owner" - "group" {
set destfile($var) $val
}
"filemode" {
if {$destfile(type) != "directory"} {
set destfile(mode) $val
}
}
"dirmode" {
if {$destfile(type) == "directory"} {
set destfile(mode) $val
}
}
}
}
#### Set the values specified for this specific file
if {[info exists fileinfo($destfile(name))]} {
array set destfile $fileinfo($destfile(name))
}
#### Take the data and format it into a list in the order specified
set destline_data [list]
foreach field $::cael::build::pkgmap_fields {
if {![info exists destfile($field)]} {
lappend destline_data ""
continue
}
lappend destline_data $destfile($field)
}
set destline [csv::join $destline_data]
puts $fd $destline
}
close $fd
## Create package information file
### XXX: TODO
set pkginfofile [file join $instmetadir info]
set fd [open $pkginfofile w]
### Domain
puts $fd "domain: $pkgdomain"
### Name
puts $fd "name: $pkgname"
### Version
puts $fd "version: $version"
### Variant
puts $fd "variant: $variant"
### Arch
#### XXX: TODO: Figure this out somehow
puts $fd "arch: ..."
### Release
#### XXX: TODO: Need to pull this from the build database
puts $fd "release: ..."
### Flags
#### XXX: TODO: Figure out how this will be formatted
puts $fd "flags: ..."
### Compiler (if not static)
#### XXX: TODO: Needed ? Could this be a flag (.e.g, compiler=blah) ?
close $fd
## Create Dependencies list file
### XXX: TODO
## Create Conflict list file
### XXX: TODO
}
proc ::cael::build::build {pkgname pkgdomain {version "latest"} {variant "default"}} {
# Parse package data
array set pkginfo [_parse_pkg $pkgname $version $variant]
if {[info exists pkginfo(scripts)]} {
array set scripts $pkginfo(scripts)
} else {
array set scripts [list]
}
array set versinfo $pkginfo(versinfo)
# Add missing values
if {![info exists pkginfo(env_vars)]} {
set pkginfo(env_vars) ""
}
# Create working directory
set tmpdir "/tmp"
if {[info exists ::env(TMPDIR)]} {
set tmpdir $::env(TMPDIR)
}
set random [string map [list "." ""] [expr {rand() * 10.0}][expr {rand() * 10.0}]]
set workdir [file join $tmpdir "build-${random}"]
set extradir [file join $workdir "extra"]
set pkgdir [file join $workdir "pkgroot"]
set instrootdir [file join $pkgdir "root"]
set instmetadir [file join $pkgdir "install"]
set instlogdir [file join $instmetadir "logs"]
# Create extra data, if any
set extradir_src [file join $::cael::build::pkginfodir $pkgname]
if {[file isdirectory $extradir_src]} {
file mkdir $extradir
foreach copyfile [glob -directory $extradir_src *] {
file copy -force -- $copyfile $extradir
}
lappend pkginfo(env_vars) extraroot $extradir
}
# Update package data
lappend pkginfo(env_vars) version $versinfo(version) destroot $instrootdir buildroot $workdir metadataroot $instmetadir
# Update with paths to installation
lappend pkginfo(env_vars) _APPBOX_PREFIX /apps/${pkgname}/$versinfo(version) _APPBOX_SYSCONFDIR /system/config/apps/$pkgname
# Include default scripts if needed
foreach {default_script_name default_script} $::cael::build::default_scripts {
if {![info exists scripts($default_script_name)]} {
set scripts($default_script_name) $default_script
}
}
# Create build scripts
set scripts_files_list [gen_scripts $pkgname $version $variant $workdir $instrootdir [array get versinfo] [array get pkginfo] [array get scripts]]
array set scripts_files $scripts_files_list
# Execute build scripts
## Execute immediate scripts
### XXX: TODO
## Execute scripts related to building and installing
foreach script [list compile install] {
if {![info exists scripts_files($script)]} {
continue
}
file attributes $scripts_files($script) -permissions +x
set compile_ret [exec $scripts_files($script) 2>@1]
### Log script outputs
set logfile "$scripts_files($script).log"
set fd [open $logfile w]
puts $fd $compile_ret
close $fd
}
# Create package metadata
gen_meta $pkgname $pkgdomain $version $variant $workdir $instrootdir $instmetadir [array get versinfo] [array get pkginfo] [array get scripts]
# Put package scripts in package meta-data directory
foreach script [list post-install post-remove pre-install pre-remove] {
if {![info exists scripts_files($script)]} {
continue
}
file attributes $scripts_files($script) -permissions +x
file copy $scripts_files($script) $instmetadir
}
# Put package log files in package meta-data directory
file mkdir $instlogdir
foreach {script script_file} [array get scripts_files] {
set log_file "${script_file}.log"
if {![file exists $log_file]} {
continue
}
file copy -- $log_file $instlogdir
}
# Create package file
## Create tarball
set pkgfilename "${pkgname}-$versinfo(version).tgz"
exec tar -C $pkgdir -zcf $pkgfilename .
# Cleanup
file delete -force -- $workdir
}
foreach package $packages {
::cael::build::build $package $domain
}