trunk/pkgtools/svn-cleanup-pkg-tree-sql.sh

#!/bin/bash

# export default C locale
export LC_ALL=C

MY_REPOS=( "branches/R11-stable" "trunk" )
MY_ARCHES=( "x86_64" "i686" )
MY_DISTRO_PKG_BASE="/var/www/htdocs/magellan/R11"
MY_DISTRO_HTTP_PKG_BASE="http://magellan-linux.de/magellan/R11"
MY_SSHLOGIN_BASE="tjoke@magellan-linux.de"
MY_MAGE_FEATURES=( "pkgdistrotag" )
MY_MAGE_FEATURES_i686=( "srcpkgbuild" )

# decide wether to use http method or not
USE_HTTP=0
if [[ $1 = --http ]]
then
	USE_HTTP=1
	shift
fi

MY_SVN_BASE="$1"
if [[ -z ${MY_SVN_BASE} ]]
then
	MY_SVN_BASE="$(mktemp -d -t tree-cleanup.XXXXXXXXXX)"
fi
MY_SVN_OPTS=""
MY_SVN_ROOT="svn://cvs.magellan-linux.de"

DBFILE="${MY_SVN_BASE}/pkgdb"

sql()
{
	local sqlcmd="$*"
	[[ -z ${sqlcmd} ]] && die "no sqlcmd given."

	sqlite3 -nullvalue 'NULL' -list -separator '|' ${DBFILE} << EOF || die "error running '$@'"
${sqlcmd};
EOF
}

# load_arch_values ARCH
load_arch_values()
{
	local arch="$1"

	MAGE_FEATURES=( ${MY_MAGE_FEATURES[*]} )
	MAGE_FEATURES+=( $(eval echo \${MY_MAGE_FEATURES_${arch}[*]}) )
	export MAGE_FEATURES

	export ARCH="${arch}"
	export CHOST="${arch}-pc-linux-gnu"
	export PACKAGES_SERVER_PATH="packages/${arch}"
	export PKGDIR="${MY_DISTRO_PKG_BASE}/packages/${arch}/"
	export HTTP_PKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/${arch}/"
	export SRCPKGDIR="${MY_DISTRO_PKG_BASE}/packages/src/"
	export HTTP_SRCPKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/src/"
}

# generate_pkg_file_list REPO
generate_pkg_file_list()
{
	local repo="$1"
	local smage
	local arch
	local _findlist
	local subpackage
	export SMAGESCRIPTSDIR="${repo}"

	_findlist=$(find ${repo} -name \*.${SMAGESUFFIX} | sort -u)
	for arch in ${MY_ARCHES[*]}
	do
		load_arch_values ${arch}
		load_mage_features

		for smage in ${_findlist}
		do
			echo "Processing ${smage}"
			smagesource ${smage} &> /dev/null || die

			# honor split-packages
			if [[ -n ${SPLIT_PACKAGES} ]]
			then
				local split_pkg_base="${PNAME}"
				split_save_variables
				for subpackage in ${SPLIT_PACKAGES}
				do
					export PNAME="${subpackage}"
					split_info_${subpackage}
					# fix PCATEGORIE -> PCAT
					if [[ ! -z ${PCATEGORIE} ]]
					then
						PCAT="${PCATEGORIE}"
						unset PCATEGORIE
					fi

#					echo "DEBUG: split_package()"
#					echo "DEBUG: PNAME='${PNAME}'"
#					echo "DEBUG: PVER='${PVER}'"
#					echo "DEBUG: PBUILD='${PBUILD}'"
#					echo "DEBUG: SMAGESCRIPTSDIR='${SMAGESCRIPTSDIR}'"
#					echo "DEBUG: DISTROTAG='${DISTROTAG}'"
#					echo "DEBUG: print_distrotag()='$(print_distrotag)'"
#					echo "DEBUG: ARCH='${ARCH}'"
#					echo "DEBUG: CHOST='${CHOST}'"
#					echo "DEBUG: PACKAGES_SERVER_PATH='${PACKAGES_SERVER_PATH}'"
#					echo "DEBUG: PKGDIR='${PKGDIR}'"
#					echo "DEBUG: SRCPKGDIR='${SRCPKGDIR}'"

					sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values(
						'${PNAME}',
						'${PVER}',
						'${PBUILD}',
						'${ARCH}',
						'$(print_distrotag)',
						'${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')"

					# restore smage environment
					split_restore_variables
				done
				# unset all saved smage variables
				split_unset_variables
				unset split_pkg_base
			else
				sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values(
					'${PNAME}',
					'${PVER}',
					'${PBUILD}',
					'${ARCH}',
					'$(print_distrotag)',
					'${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')"
			fi
			if mqueryfeature "srcpkgbuild"
			then
				sql "insert or ignore into srcpkgs(pname, pver, pbuild, filename ) values(
					'${PNAME}',
					'${PVER}',
					'${PBUILD}',
					'${PNAME}${target}-${PVER}-${PBUILD}.${SRCPKGSUFFIX}')"
			fi
			unset PNAME
			unset PVER
			unset PBUILD
			unset SPLIT_PACKAGES
		done
	done
}

# check_pkglist TABLE PKGFILE
check_pkglist()
{
	local table="$1"
	local pkg="$2"

	if [[ -z $(sql "select filename from ${table} where filename='${pkg}'") ]]
	then
		return 1
	fi
	return 0
}

# checkout svn sources for mage and smage repos
export SVNROOT="${MY_SVN_ROOT}"
[[ ! -d ${MY_SVN_BASE} ]] && install -d ${MY_SVN_BASE}
for module in magellan-source/trunk/mage ${MY_REPOS[*]}
do
	case ${module} in
		magellan-source/*) repo="${module}"; codir=mage-src ;;
		*) repo="smage/${module}"; codir="smage/${module}" ;;
	esac

	if [ ! -d ${MY_SVN_BASE}/${module}/.svn ]
	then
		( cd ${MY_SVN_BASE}; svn checkout ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} )
	else
		( cd ${MY_SVN_BASE}; svn update ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} )
	fi

	# exclude deprecated
	if [ -d ${MY_SVN_BASE}/${codir}/deprecated ]
	then
		echo "deleting ${MY_SVN_BASE}/${codir}/deprecated"
		rm -r ${MY_SVN_BASE}/${codir}/deprecated
	fi
done

# include functions from fresh mage checkout
source ${MY_SVN_BASE}/mage-src/usr/lib/mage/mage.rc.global.in
export MLIBDIR="${MY_SVN_BASE}/mage-src/usr/lib/mage"
source ${MLIBDIR}/mage4.functions.sh
source ${MLIBDIR}/smage2.functions.sh

sql "create table IF NOT EXISTS pkgs(id integer primary key,
		pname text,
		pver text,
		pbuild text,
		arch text,
		distrotag text,
		filename text unique)"
sql "create table IF NOT EXISTS srcpkgs(id integer primary key,
		pname text,
		pver text,
		pbuild text,
		filename text unique)"

for repo in ${MY_REPOS[*]}
do
	generate_pkg_file_list "${MY_SVN_BASE}/smage/${repo}"
done

sql "select filename from pkgs"
sql "select filename from srcpkgs"

# initialize lists
if [[ ${USE_HTTP} = 1 ]]
then
	:> ${MY_SVN_BASE}/pkglist
	:> ${MY_SVN_BASE}/srcpkglist
fi

for arch in ${MY_ARCHES[*]}
do
	load_arch_values "${arch}"
	load_mage_features

	if [[ ${USE_HTTP} = 1 ]]
	then
		_filelist=$(lynx -dump -listonly ${HTTP_PKGDIR} | grep ".${PKGSUFFIX}" | sed "s:.*${PACKAGES_SERVER_PATH}/\(.*.${PKGSUFFIX}\):\1:")
	else
		_filelist=$(find ${PKGDIR} -name \*.${PKGSUFFIX} -printf '%f\n' | sort)
	fi

	if [[ ${USE_HTTP} = 1 ]]
	then
		echo "install -d ${PKGDIR}/_deprecated" >> ${MY_SVN_BASE}/pkglist
	else
		if [ ! -d ${PKGDIR}/_deprecated ]
		then
			install -d ${PKGDIR}/_deprecated || die "install ${PKGDIR}/${arch}/_deprecated"
		fi
	fi
	for file in ${_filelist}
	do
		if check_pkglist pkgs "${file}"
		then
			echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}"
		else
			echo -e "${COLRED}[d] - ${file}${COLDEFAULT}"
			if [[ ${USE_HTTP} = 1 ]]
			then
				echo "mv -v ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ " >> ${MY_SVN_BASE}/pkglist
			else
				mv ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ || die "move ${PKGDIR}/${file} -> _deprecated"
			fi
		fi
	done
	if mqueryfeature "srcpkgbuild"
	then
		if [[ ${USE_HTTP} = 1 ]]
		then
			_filelist=$(lynx -dump -listonly ${HTTP_SRCPKGDIR} | grep ".${SRCPKGSUFFIX}" | sed "s:.*packages/src/\(.*.${SRCPKGSUFFIX}\):\1:")
		else
			_filelist=$(find ${SRCPKGDIR} -name \*.${SRCPKGSUFFIX} -printf '%f\n' | sort)
		fi

		if [[ ${USE_HTTP} = 1 ]]
		then
			echo "install -d ${SRCPKGDIR}/_deprecated" >> ${MY_SVN_BASE}/srcpkglist
		else
			if [ ! -d ${SRCPKGDIR}/_deprecated ]
			then
				install -d ${SRCPKGDIR}/_deprecated || die "install ${SRCPKGDIR}/_deprecated"
			fi
		fi
		for file in ${_filelist}
		do
			if check_pkglist srcpkgs "${file}"
			then
				echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}"
			else
				echo -e "${COLRED}[d] - ${file}${COLDEFAULT}"
				if [[ ${USE_HTTP} = 1 ]]
				then
					echo "mv -v ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/" >> ${MY_SVN_BASE}/srcpkglist
				else
					mv ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/ || die "move ${SRCPKGDIR}/${file} -> _deprecated"
				fi
			fi
		done
	fi
done

if [[ ${USE_HTTP} =  1 ]]
then
	for list in pkglist srcpkglist
	do
		if [[ -f ${MY_SVN_BASE}/${list} ]]
		then
			scp ${MY_SVN_BASE}/${list} ${MY_SSHLOGIN_BASE}:~
			ssh ${MY_SSHLOGIN_BASE} bash ~/${list}
		fi
	done
fi
1	niro	2108	#!/bin/bash
2
3			# export default C locale
4			export LC_ALL=C
5
6			MY_REPOS=( "branches/R11-stable" "trunk" )
7			MY_ARCHES=( "x86_64" "i686" )
8			MY_DISTRO_PKG_BASE="/var/www/htdocs/magellan/R11"
9			MY_DISTRO_HTTP_PKG_BASE="http://magellan-linux.de/magellan/R11"
10			MY_SSHLOGIN_BASE="tjoke@magellan-linux.de"
11			MY_MAGE_FEATURES=( "pkgdistrotag" )
12			MY_MAGE_FEATURES_i686=( "srcpkgbuild" )
13
14			# decide wether to use http method or not
15			USE_HTTP=0
16			if [[ $1 = --http ]]
17			then
18			USE_HTTP=1
19			shift
20			fi
21
22			MY_SVN_BASE="$1"
23			if [[ -z ${MY_SVN_BASE} ]]
24			then
25			MY_SVN_BASE="$(mktemp -d -t tree-cleanup.XXXXXXXXXX)"
26			fi
27			MY_SVN_OPTS=""
28			MY_SVN_ROOT="svn://cvs.magellan-linux.de"
29
30			DBFILE="${MY_SVN_BASE}/pkgdb"
31
32			sql()
33			{
34			local sqlcmd="$*"
35			[[ -z ${sqlcmd} ]] && die "no sqlcmd given."
36
37			sqlite3 -nullvalue 'NULL' -list -separator '\|' ${DBFILE} << EOF \|\| die "error running '$@'"
38			${sqlcmd};
39			EOF
40			}
41
42			# load_arch_values ARCH
43			load_arch_values()
44			{
45			local arch="$1"
46
47			MAGE_FEATURES=( ${MY_MAGE_FEATURES[*]} )
48			MAGE_FEATURES+=( $(eval echo \${MY_MAGE_FEATURES_${arch}[*]}) )
49			export MAGE_FEATURES
50
51			export ARCH="${arch}"
52			export CHOST="${arch}-pc-linux-gnu"
53			export PACKAGES_SERVER_PATH="packages/${arch}"
54			export PKGDIR="${MY_DISTRO_PKG_BASE}/packages/${arch}/"
55			export HTTP_PKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/${arch}/"
56			export SRCPKGDIR="${MY_DISTRO_PKG_BASE}/packages/src/"
57			export HTTP_SRCPKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/src/"
58			}
59
60			# generate_pkg_file_list REPO
61			generate_pkg_file_list()
62			{
63			local repo="$1"
64			local smage
65			local arch
66			local _findlist
67			local subpackage
68			export SMAGESCRIPTSDIR="${repo}"
69
70			_findlist=$(find ${repo} -name \*.${SMAGESUFFIX} \| sort -u)
71			for arch in ${MY_ARCHES[*]}
72			do
73			load_arch_values ${arch}
74			load_mage_features
75
76			for smage in ${_findlist}
77			do
78			echo "Processing ${smage}"
79			smagesource ${smage} &> /dev/null \|\| die
80
81			# honor split-packages
82			if [[ -n ${SPLIT_PACKAGES} ]]
83			then
84			local split_pkg_base="${PNAME}"
85			split_save_variables
86			for subpackage in ${SPLIT_PACKAGES}
87			do
88			export PNAME="${subpackage}"
89			split_info_${subpackage}
90			# fix PCATEGORIE -> PCAT
91			if [[ ! -z ${PCATEGORIE} ]]
92			then
93			PCAT="${PCATEGORIE}"
94			unset PCATEGORIE
95			fi
96
97			# echo "DEBUG: split_package()"
98			# echo "DEBUG: PNAME='${PNAME}'"
99			# echo "DEBUG: PVER='${PVER}'"
100			# echo "DEBUG: PBUILD='${PBUILD}'"
101			# echo "DEBUG: SMAGESCRIPTSDIR='${SMAGESCRIPTSDIR}'"
102			# echo "DEBUG: DISTROTAG='${DISTROTAG}'"
103			# echo "DEBUG: print_distrotag()='$(print_distrotag)'"
104			# echo "DEBUG: ARCH='${ARCH}'"
105			# echo "DEBUG: CHOST='${CHOST}'"
106			# echo "DEBUG: PACKAGES_SERVER_PATH='${PACKAGES_SERVER_PATH}'"
107			# echo "DEBUG: PKGDIR='${PKGDIR}'"
108			# echo "DEBUG: SRCPKGDIR='${SRCPKGDIR}'"
109
110			sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values(
111			'${PNAME}',
112			'${PVER}',
113			'${PBUILD}',
114			'${ARCH}',
115			'$(print_distrotag)',
116			'${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')"
117
118			# restore smage environment
119			split_restore_variables
120			done
121			# unset all saved smage variables
122			split_unset_variables
123			unset split_pkg_base
124			else
125			sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values(
126			'${PNAME}',
127			'${PVER}',
128			'${PBUILD}',
129			'${ARCH}',
130			'$(print_distrotag)',
131			'${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')"
132			fi
133			if mqueryfeature "srcpkgbuild"
134			then
135			sql "insert or ignore into srcpkgs(pname, pver, pbuild, filename ) values(
136			'${PNAME}',
137			'${PVER}',
138			'${PBUILD}',
139			'${PNAME}${target}-${PVER}-${PBUILD}.${SRCPKGSUFFIX}')"
140			fi
141			unset PNAME
142			unset PVER
143			unset PBUILD
144			unset SPLIT_PACKAGES
145			done
146			done
147			}
148
149			# check_pkglist TABLE PKGFILE
150			check_pkglist()
151			{
152			local table="$1"
153			local pkg="$2"
154
155			if [[ -z $(sql "select filename from ${table} where filename='${pkg}'") ]]
156			then
157			return 1
158			fi
159			return 0
160			}
161
162			# checkout svn sources for mage and smage repos
163			export SVNROOT="${MY_SVN_ROOT}"
164			[[ ! -d ${MY_SVN_BASE} ]] && install -d ${MY_SVN_BASE}
165			for module in magellan-source/trunk/mage ${MY_REPOS[*]}
166			do
167			case ${module} in
168			magellan-source/*) repo="${module}"; codir=mage-src ;;
169			*) repo="smage/${module}"; codir="smage/${module}" ;;
170			esac
171
172			if [ ! -d ${MY_SVN_BASE}/${module}/.svn ]
173			then
174			( cd ${MY_SVN_BASE}; svn checkout ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} )
175			else
176			( cd ${MY_SVN_BASE}; svn update ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} )
177			fi
178
179			# exclude deprecated
180			if [ -d ${MY_SVN_BASE}/${codir}/deprecated ]
181			then
182			echo "deleting ${MY_SVN_BASE}/${codir}/deprecated"
183			rm -r ${MY_SVN_BASE}/${codir}/deprecated
184			fi
185			done
186
187			# include functions from fresh mage checkout
188			source ${MY_SVN_BASE}/mage-src/usr/lib/mage/mage.rc.global.in
189			export MLIBDIR="${MY_SVN_BASE}/mage-src/usr/lib/mage"
190			source ${MLIBDIR}/mage4.functions.sh
191			source ${MLIBDIR}/smage2.functions.sh
192
193			sql "create table IF NOT EXISTS pkgs(id integer primary key,
194			pname text,
195			pver text,
196			pbuild text,
197			arch text,
198			distrotag text,
199			filename text unique)"
200			sql "create table IF NOT EXISTS srcpkgs(id integer primary key,
201			pname text,
202			pver text,
203			pbuild text,
204			filename text unique)"
205
206			for repo in ${MY_REPOS[*]}
207			do
208			generate_pkg_file_list "${MY_SVN_BASE}/smage/${repo}"
209			done
210
211			sql "select filename from pkgs"
212			sql "select filename from srcpkgs"
213
214			# initialize lists
215			if [[ ${USE_HTTP} = 1 ]]
216			then
217			:> ${MY_SVN_BASE}/pkglist
218			:> ${MY_SVN_BASE}/srcpkglist
219			fi
220
221			for arch in ${MY_ARCHES[*]}
222			do
223			load_arch_values "${arch}"
224			load_mage_features
225
226			if [[ ${USE_HTTP} = 1 ]]
227			then
228			_filelist=$(lynx -dump -listonly ${HTTP_PKGDIR} \| grep ".${PKGSUFFIX}" \| sed "s:.${PACKAGES_SERVER_PATH}/\(..${PKGSUFFIX}\):\1:")
229			else
230			_filelist=$(find ${PKGDIR} -name \*.${PKGSUFFIX} -printf '%f\n' \| sort)
231			fi
232
233			if [[ ${USE_HTTP} = 1 ]]
234			then
235			echo "install -d ${PKGDIR}/_deprecated" >> ${MY_SVN_BASE}/pkglist
236			else
237			if [ ! -d ${PKGDIR}/_deprecated ]
238			then
239			install -d ${PKGDIR}/_deprecated \|\| die "install ${PKGDIR}/${arch}/_deprecated"
240			fi
241			fi
242			for file in ${_filelist}
243			do
244			if check_pkglist pkgs "${file}"
245			then
246			echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}"
247			else
248			echo -e "${COLRED}[d] - ${file}${COLDEFAULT}"
249			if [[ ${USE_HTTP} = 1 ]]
250			then
251			echo "mv -v ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ " >> ${MY_SVN_BASE}/pkglist
252			else
253			mv ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ \|\| die "move ${PKGDIR}/${file} -> _deprecated"
254			fi
255			fi
256			done
257			if mqueryfeature "srcpkgbuild"
258			then
259			if [[ ${USE_HTTP} = 1 ]]
260			then
261			_filelist=$(lynx -dump -listonly ${HTTP_SRCPKGDIR} \| grep ".${SRCPKGSUFFIX}" \| sed "s:.packages/src/\(..${SRCPKGSUFFIX}\):\1:")
262			else
263			_filelist=$(find ${SRCPKGDIR} -name \*.${SRCPKGSUFFIX} -printf '%f\n' \| sort)
264			fi
265
266			if [[ ${USE_HTTP} = 1 ]]
267			then
268			echo "install -d ${SRCPKGDIR}/_deprecated" >> ${MY_SVN_BASE}/srcpkglist
269			else
270			if [ ! -d ${SRCPKGDIR}/_deprecated ]
271			then
272			install -d ${SRCPKGDIR}/_deprecated \|\| die "install ${SRCPKGDIR}/_deprecated"
273			fi
274			fi
275			for file in ${_filelist}
276			do
277			if check_pkglist srcpkgs "${file}"
278			then
279			echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}"
280			else
281			echo -e "${COLRED}[d] - ${file}${COLDEFAULT}"
282			if [[ ${USE_HTTP} = 1 ]]
283			then
284			echo "mv -v ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/" >> ${MY_SVN_BASE}/srcpkglist
285			else
286			mv ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/ \|\| die "move ${SRCPKGDIR}/${file} -> _deprecated"
287			fi
288			fi
289			done
290			fi
291			done
292
293			if [[ ${USE_HTTP} = 1 ]]
294			then
295			for list in pkglist srcpkglist
296			do
297			if [[ -f ${MY_SVN_BASE}/${list} ]]
298			then
299			scp ${MY_SVN_BASE}/${list} ${MY_SSHLOGIN_BASE}:~
300			ssh ${MY_SSHLOGIN_BASE} bash ~/${list}
301			fi
302			done
303			fi