trunk/pkgtools/svn-cleanup-pkg-tree-sql.sh

#!/bin/bash

# export default C locale
export LC_ALL=C

MY_REPOS=( "branches/R11-stable" "trunk" )
MY_ARCHES=( "x86_64" "i686" )
MY_DISTRO_PKG_BASE="/var/www/htdocs/magellan/R11"
MY_DISTRO_HTTP_PKG_BASE="http://magellan-linux.de/magellan/R11"
MY_SSHLOGIN_BASE="tjoke@magellan-linux.de"
MY_MAGE_FEATURES=( "pkgdistrotag" )
MY_MAGE_FEATURES_i686=( "srcpkgbuild" )

# decide wether to use http method or not
USE_HTTP=0
if [[ $1 = --http ]]
then
	USE_HTTP=1
	shift
fi

MY_SVN_BASE="$1"
if [[ -z ${MY_SVN_BASE} ]]
then
	MY_SVN_BASE="$(mktemp -d -t tree-cleanup.XXXXXXXXXX)"
fi
MY_SVN_OPTS=""
MY_SVN_ROOT="svn://cvs.magellan-linux.de"

DBFILE="${MY_SVN_BASE}/pkgdb"

sql()
{
	local sqlcmd="$*"
	[[ -z ${sqlcmd} ]] && die "no sqlcmd given."

	sqlite3 -nullvalue 'NULL' -list -separator '|' ${DBFILE} << EOF || die "error running '$@'"
${sqlcmd};
EOF
}

# load_arch_values ARCH
load_arch_values()
{
	local arch="$1"

	MAGE_FEATURES=( ${MY_MAGE_FEATURES[*]} )
	MAGE_FEATURES+=( $(eval echo \${MY_MAGE_FEATURES_${arch}[*]}) )
	export MAGE_FEATURES

	export ARCH="${arch}"
	export CHOST="${arch}-pc-linux-gnu"
	export PACKAGES_SERVER_PATH="packages/${arch}"
	export PKGDIR="${MY_DISTRO_PKG_BASE}/packages/${arch}/"
	export HTTP_PKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/${arch}/"
	export SRCPKGDIR="${MY_DISTRO_PKG_BASE}/packages/src/"
	export HTTP_SRCPKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/src/"
}

# generate_pkg_file_list REPO
generate_pkg_file_list()
{
	local repo="$1"
	local smage
	local arch
	local _findlist
	local subpackage
	export SMAGESCRIPTSDIR="${repo}"

	_findlist=$(find ${repo} -name \*.${SMAGESUFFIX} | sort -u)
	for arch in ${MY_ARCHES[*]}
	do
		load_arch_values ${arch}
		load_mage_features

		for smage in ${_findlist}
		do
			echo "Processing ${smage}"
			smagesource ${smage} &> /dev/null || die

			# honor split-packages
			if [[ -n ${SPLIT_PACKAGES} ]]
			then
				local split_pkg_base="${PNAME}"
				split_save_variables
				for subpackage in ${SPLIT_PACKAGES}
				do
					export PNAME="${subpackage}"
					split_info_${subpackage}
					# fix PCATEGORIE -> PCAT
					if [[ ! -z ${PCATEGORIE} ]]
					then
						PCAT="${PCATEGORIE}"
						unset PCATEGORIE
					fi

#					echo "DEBUG: split_package()"
#					echo "DEBUG: PNAME='${PNAME}'"
#					echo "DEBUG: PVER='${PVER}'"
#					echo "DEBUG: PBUILD='${PBUILD}'"
#					echo "DEBUG: SMAGESCRIPTSDIR='${SMAGESCRIPTSDIR}'"
#					echo "DEBUG: DISTROTAG='${DISTROTAG}'"
#					echo "DEBUG: print_distrotag()='$(print_distrotag)'"
#					echo "DEBUG: ARCH='${ARCH}'"
#					echo "DEBUG: CHOST='${CHOST}'"
#					echo "DEBUG: PACKAGES_SERVER_PATH='${PACKAGES_SERVER_PATH}'"
#					echo "DEBUG: PKGDIR='${PKGDIR}'"
#					echo "DEBUG: SRCPKGDIR='${SRCPKGDIR}'"

					sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values(
						'${PNAME}',
						'${PVER}',
						'${PBUILD}',
						'${ARCH}',
						'$(print_distrotag)',
						'${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')"

					# restore smage environment
					split_restore_variables
				done
				# unset all saved smage variables
				split_unset_variables
				unset split_pkg_base
			else
				sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values(
					'${PNAME}',
					'${PVER}',
					'${PBUILD}',
					'${ARCH}',
					'$(print_distrotag)',
					'${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')"
			fi
			if mqueryfeature "srcpkgbuild"
			then
				sql "insert or ignore into srcpkgs(pname, pver, pbuild, filename ) values(
					'${PNAME}',
					'${PVER}',
					'${PBUILD}',
					'${PNAME}${target}-${PVER}-${PBUILD}.${SRCPKGSUFFIX}')"
			fi
			unset PNAME
			unset PVER
			unset PBUILD
			unset SPLIT_PACKAGES
		done
	done
}

# check_pkglist TABLE PKGFILE
check_pkglist()
{
	local table="$1"
	local pkg="$2"

	if [[ -z $(sql "select filename from ${table} where filename='${pkg}'") ]]
	then
		return 1
	fi
	return 0
}

# checkout svn sources for mage and smage repos
export SVNROOT="${MY_SVN_ROOT}"
[[ ! -d ${MY_SVN_BASE} ]] && install -d ${MY_SVN_BASE}
for module in magellan-source/trunk/mage ${MY_REPOS[*]}
do
	case ${module} in
		magellan-source/*) repo="${module}"; codir=mage-src ;;
		*) repo="smage/${module}"; codir="smage/${module}" ;;
	esac

	if [ ! -d ${MY_SVN_BASE}/${module}/.svn ]
	then
		( cd ${MY_SVN_BASE}; svn checkout ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} )
	else
		( cd ${MY_SVN_BASE}; svn update ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} )
	fi

	# exclude deprecated
	if [ -d ${MY_SVN_BASE}/${codir}/deprecated ]
	then
		echo "deleting ${MY_SVN_BASE}/${codir}/deprecated"
		rm -r ${MY_SVN_BASE}/${codir}/deprecated
	fi
done

# include functions from fresh mage checkout
source ${MY_SVN_BASE}/mage-src/usr/lib/mage/mage.rc.global.in
export MLIBDIR="${MY_SVN_BASE}/mage-src/usr/lib/mage"
source ${MLIBDIR}/mage4.functions.sh
source ${MLIBDIR}/smage2.functions.sh

sql "create table IF NOT EXISTS pkgs(id integer primary key,
		pname text,
		pver text,
		pbuild text,
		arch text,
		distrotag text,
		filename text unique)"
sql "create table IF NOT EXISTS srcpkgs(id integer primary key,
		pname text,
		pver text,
		pbuild text,
		filename text unique)"

for repo in ${MY_REPOS[*]}
do
	generate_pkg_file_list "${MY_SVN_BASE}/smage/${repo}"
done

sql "select filename from pkgs"
sql "select filename from srcpkgs"

# initialize lists
if [[ ${USE_HTTP} = 1 ]]
then
	:> ${MY_SVN_BASE}/pkglist
	:> ${MY_SVN_BASE}/srcpkglist
fi

for arch in ${MY_ARCHES[*]}
do
	load_arch_values "${arch}"
	load_mage_features

	if [[ ${USE_HTTP} = 1 ]]
	then
		_filelist=$(lynx -dump -listonly ${HTTP_PKGDIR} | grep ".${PKGSUFFIX}" | sed "s:.*${PACKAGES_SERVER_PATH}/\(.*.${PKGSUFFIX}\):\1:")
	else
		_filelist=$(find ${PKGDIR} -name \*.${PKGSUFFIX} -printf '%f\n' | sort)
	fi

	if [[ ${USE_HTTP} = 1 ]]
	then
		echo "install -d ${PKGDIR}/_deprecated" >> ${MY_SVN_BASE}/pkglist
	else
		if [ ! -d ${PKGDIR}/_deprecated ]
		then
			install -d ${PKGDIR}/_deprecated || die "install ${PKGDIR}/${arch}/_deprecated"
		fi
	fi
	for file in ${_filelist}
	do
		if check_pkglist pkgs "${file}"
		then
			echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}"
		else
			echo -e "${COLRED}[d] - ${file}${COLDEFAULT}"
			if [[ ${USE_HTTP} = 1 ]]
			then
				echo "mv -v ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ " >> ${MY_SVN_BASE}/pkglist
			else
				mv ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ || die "move ${PKGDIR}/${file} -> _deprecated"
			fi
		fi
	done
	if mqueryfeature "srcpkgbuild"
	then
		if [[ ${USE_HTTP} = 1 ]]
		then
			_filelist=$(lynx -dump -listonly ${HTTP_SRCPKGDIR} | grep ".${SRCPKGSUFFIX}" | sed "s:.*packages/src/\(.*.${SRCPKGSUFFIX}\):\1:")
		else
			_filelist=$(find ${SRCPKGDIR} -name \*.${SRCPKGSUFFIX} -printf '%f\n' | sort)
		fi

		if [[ ${USE_HTTP} = 1 ]]
		then
			echo "install -d ${SRCPKGDIR}/_deprecated" >> ${MY_SVN_BASE}/srcpkglist
		else
			if [ ! -d ${SRCPKGDIR}/_deprecated ]
			then
				install -d ${SRCPKGDIR}/_deprecated || die "install ${SRCPKGDIR}/_deprecated"
			fi
		fi
		for file in ${_filelist}
		do
			if check_pkglist srcpkgs "${file}"
			then
				echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}"
			else
				echo -e "${COLRED}[d] - ${file}${COLDEFAULT}"
				if [[ ${USE_HTTP} = 1 ]]
				then
					echo "mv -v ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/" >> ${MY_SVN_BASE}/srcpkglist
				else
					mv ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/ || die "move ${SRCPKGDIR}/${file} -> _deprecated"
				fi
			fi
		done
	fi
done

if [[ ${USE_HTTP} =  1 ]]
then
	for list in pkglist srcpkglist
	do
		if [[ -f ${MY_SVN_BASE}/${list} ]]
		then
			scp ${MY_SVN_BASE}/${list} ${MY_SSHLOGIN_BASE}:~
			ssh ${MY_SSHLOGIN_BASE} bash ~/${list}
		fi
	done
fi
1	#!/bin/bash
2
3	# export default C locale
4	export LC_ALL=C
5
6	MY_REPOS=( "branches/R11-stable" "trunk" )
7	MY_ARCHES=( "x86_64" "i686" )
8	MY_DISTRO_PKG_BASE="/var/www/htdocs/magellan/R11"
9	MY_DISTRO_HTTP_PKG_BASE="http://magellan-linux.de/magellan/R11"
10	MY_SSHLOGIN_BASE="tjoke@magellan-linux.de"
11	MY_MAGE_FEATURES=( "pkgdistrotag" )
12	MY_MAGE_FEATURES_i686=( "srcpkgbuild" )
13
14	# decide wether to use http method or not
15	USE_HTTP=0
16	if [[ $1 = --http ]]
17	then
18	USE_HTTP=1
19	shift
20	fi
21
22	MY_SVN_BASE="$1"
23	if [[ -z ${MY_SVN_BASE} ]]
24	then
25	MY_SVN_BASE="$(mktemp -d -t tree-cleanup.XXXXXXXXXX)"
26	fi
27	MY_SVN_OPTS=""
28	MY_SVN_ROOT="svn://cvs.magellan-linux.de"
29
30	DBFILE="${MY_SVN_BASE}/pkgdb"
31
32	sql()
33	{
34	local sqlcmd="$*"
35	[[ -z ${sqlcmd} ]] && die "no sqlcmd given."
36
37	sqlite3 -nullvalue 'NULL' -list -separator '\|' ${DBFILE} << EOF \|\| die "error running '$@'"
38	${sqlcmd};
39	EOF
40	}
41
42	# load_arch_values ARCH
43	load_arch_values()
44	{
45	local arch="$1"
46
47	MAGE_FEATURES=( ${MY_MAGE_FEATURES[*]} )
48	MAGE_FEATURES+=( $(eval echo \${MY_MAGE_FEATURES_${arch}[*]}) )
49	export MAGE_FEATURES
50
51	export ARCH="${arch}"
52	export CHOST="${arch}-pc-linux-gnu"
53	export PACKAGES_SERVER_PATH="packages/${arch}"
54	export PKGDIR="${MY_DISTRO_PKG_BASE}/packages/${arch}/"
55	export HTTP_PKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/${arch}/"
56	export SRCPKGDIR="${MY_DISTRO_PKG_BASE}/packages/src/"
57	export HTTP_SRCPKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/src/"
58	}
59
60	# generate_pkg_file_list REPO
61	generate_pkg_file_list()
62	{
63	local repo="$1"
64	local smage
65	local arch
66	local _findlist
67	local subpackage
68	export SMAGESCRIPTSDIR="${repo}"
69
70	_findlist=$(find ${repo} -name \*.${SMAGESUFFIX} \| sort -u)
71	for arch in ${MY_ARCHES[*]}
72	do
73	load_arch_values ${arch}
74	load_mage_features
75
76	for smage in ${_findlist}
77	do
78	echo "Processing ${smage}"
79	smagesource ${smage} &> /dev/null \|\| die
80
81	# honor split-packages
82	if [[ -n ${SPLIT_PACKAGES} ]]
83	then
84	local split_pkg_base="${PNAME}"
85	split_save_variables
86	for subpackage in ${SPLIT_PACKAGES}
87	do
88	export PNAME="${subpackage}"
89	split_info_${subpackage}
90	# fix PCATEGORIE -> PCAT
91	if [[ ! -z ${PCATEGORIE} ]]
92	then
93	PCAT="${PCATEGORIE}"
94	unset PCATEGORIE
95	fi
96
97	# echo "DEBUG: split_package()"
98	# echo "DEBUG: PNAME='${PNAME}'"
99	# echo "DEBUG: PVER='${PVER}'"
100	# echo "DEBUG: PBUILD='${PBUILD}'"
101	# echo "DEBUG: SMAGESCRIPTSDIR='${SMAGESCRIPTSDIR}'"
102	# echo "DEBUG: DISTROTAG='${DISTROTAG}'"
103	# echo "DEBUG: print_distrotag()='$(print_distrotag)'"
104	# echo "DEBUG: ARCH='${ARCH}'"
105	# echo "DEBUG: CHOST='${CHOST}'"
106	# echo "DEBUG: PACKAGES_SERVER_PATH='${PACKAGES_SERVER_PATH}'"
107	# echo "DEBUG: PKGDIR='${PKGDIR}'"
108	# echo "DEBUG: SRCPKGDIR='${SRCPKGDIR}'"
109
110	sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values(
111	'${PNAME}',
112	'${PVER}',
113	'${PBUILD}',
114	'${ARCH}',
115	'$(print_distrotag)',
116	'${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')"
117
118	# restore smage environment
119	split_restore_variables
120	done
121	# unset all saved smage variables
122	split_unset_variables
123	unset split_pkg_base
124	else
125	sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values(
126	'${PNAME}',
127	'${PVER}',
128	'${PBUILD}',
129	'${ARCH}',
130	'$(print_distrotag)',
131	'${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')"
132	fi
133	if mqueryfeature "srcpkgbuild"
134	then
135	sql "insert or ignore into srcpkgs(pname, pver, pbuild, filename ) values(
136	'${PNAME}',
137	'${PVER}',
138	'${PBUILD}',
139	'${PNAME}${target}-${PVER}-${PBUILD}.${SRCPKGSUFFIX}')"
140	fi
141	unset PNAME
142	unset PVER
143	unset PBUILD
144	unset SPLIT_PACKAGES
145	done
146	done
147	}
148
149	# check_pkglist TABLE PKGFILE
150	check_pkglist()
151	{
152	local table="$1"
153	local pkg="$2"
154
155	if [[ -z $(sql "select filename from ${table} where filename='${pkg}'") ]]
156	then
157	return 1
158	fi
159	return 0
160	}
161
162	# checkout svn sources for mage and smage repos
163	export SVNROOT="${MY_SVN_ROOT}"
164	[[ ! -d ${MY_SVN_BASE} ]] && install -d ${MY_SVN_BASE}
165	for module in magellan-source/trunk/mage ${MY_REPOS[*]}
166	do
167	case ${module} in
168	magellan-source/*) repo="${module}"; codir=mage-src ;;
169	*) repo="smage/${module}"; codir="smage/${module}" ;;
170	esac
171
172	if [ ! -d ${MY_SVN_BASE}/${module}/.svn ]
173	then
174	( cd ${MY_SVN_BASE}; svn checkout ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} )
175	else
176	( cd ${MY_SVN_BASE}; svn update ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} )
177	fi
178
179	# exclude deprecated
180	if [ -d ${MY_SVN_BASE}/${codir}/deprecated ]
181	then
182	echo "deleting ${MY_SVN_BASE}/${codir}/deprecated"
183	rm -r ${MY_SVN_BASE}/${codir}/deprecated
184	fi
185	done
186
187	# include functions from fresh mage checkout
188	source ${MY_SVN_BASE}/mage-src/usr/lib/mage/mage.rc.global.in
189	export MLIBDIR="${MY_SVN_BASE}/mage-src/usr/lib/mage"
190	source ${MLIBDIR}/mage4.functions.sh
191	source ${MLIBDIR}/smage2.functions.sh
192
193	sql "create table IF NOT EXISTS pkgs(id integer primary key,
194	pname text,
195	pver text,
196	pbuild text,
197	arch text,
198	distrotag text,
199	filename text unique)"
200	sql "create table IF NOT EXISTS srcpkgs(id integer primary key,
201	pname text,
202	pver text,
203	pbuild text,
204	filename text unique)"
205
206	for repo in ${MY_REPOS[*]}
207	do
208	generate_pkg_file_list "${MY_SVN_BASE}/smage/${repo}"
209	done
210
211	sql "select filename from pkgs"
212	sql "select filename from srcpkgs"
213
214	# initialize lists
215	if [[ ${USE_HTTP} = 1 ]]
216	then
217	:> ${MY_SVN_BASE}/pkglist
218	:> ${MY_SVN_BASE}/srcpkglist
219	fi
220
221	for arch in ${MY_ARCHES[*]}
222	do
223	load_arch_values "${arch}"
224	load_mage_features
225
226	if [[ ${USE_HTTP} = 1 ]]
227	then
228	_filelist=$(lynx -dump -listonly ${HTTP_PKGDIR} \| grep ".${PKGSUFFIX}" \| sed "s:.${PACKAGES_SERVER_PATH}/\(..${PKGSUFFIX}\):\1:")
229	else
230	_filelist=$(find ${PKGDIR} -name \*.${PKGSUFFIX} -printf '%f\n' \| sort)
231	fi
232
233	if [[ ${USE_HTTP} = 1 ]]
234	then
235	echo "install -d ${PKGDIR}/_deprecated" >> ${MY_SVN_BASE}/pkglist
236	else
237	if [ ! -d ${PKGDIR}/_deprecated ]
238	then
239	install -d ${PKGDIR}/_deprecated \|\| die "install ${PKGDIR}/${arch}/_deprecated"
240	fi
241	fi
242	for file in ${_filelist}
243	do
244	if check_pkglist pkgs "${file}"
245	then
246	echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}"
247	else
248	echo -e "${COLRED}[d] - ${file}${COLDEFAULT}"
249	if [[ ${USE_HTTP} = 1 ]]
250	then
251	echo "mv -v ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ " >> ${MY_SVN_BASE}/pkglist
252	else
253	mv ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ \|\| die "move ${PKGDIR}/${file} -> _deprecated"
254	fi
255	fi
256	done
257	if mqueryfeature "srcpkgbuild"
258	then
259	if [[ ${USE_HTTP} = 1 ]]
260	then
261	_filelist=$(lynx -dump -listonly ${HTTP_SRCPKGDIR} \| grep ".${SRCPKGSUFFIX}" \| sed "s:.packages/src/\(..${SRCPKGSUFFIX}\):\1:")
262	else
263	_filelist=$(find ${SRCPKGDIR} -name \*.${SRCPKGSUFFIX} -printf '%f\n' \| sort)
264	fi
265
266	if [[ ${USE_HTTP} = 1 ]]
267	then
268	echo "install -d ${SRCPKGDIR}/_deprecated" >> ${MY_SVN_BASE}/srcpkglist
269	else
270	if [ ! -d ${SRCPKGDIR}/_deprecated ]
271	then
272	install -d ${SRCPKGDIR}/_deprecated \|\| die "install ${SRCPKGDIR}/_deprecated"
273	fi
274	fi
275	for file in ${_filelist}
276	do
277	if check_pkglist srcpkgs "${file}"
278	then
279	echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}"
280	else
281	echo -e "${COLRED}[d] - ${file}${COLDEFAULT}"
282	if [[ ${USE_HTTP} = 1 ]]
283	then
284	echo "mv -v ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/" >> ${MY_SVN_BASE}/srcpkglist
285	else
286	mv ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/ \|\| die "move ${SRCPKGDIR}/${file} -> _deprecated"
287	fi
288	fi
289	done
290	fi
291	done
292
293	if [[ ${USE_HTTP} = 1 ]]
294	then
295	for list in pkglist srcpkglist
296	do
297	if [[ -f ${MY_SVN_BASE}/${list} ]]
298	then
299	scp ${MY_SVN_BASE}/${list} ${MY_SSHLOGIN_BASE}:~
300	ssh ${MY_SSHLOGIN_BASE} bash ~/${list}
301	fi
302	done
303	fi