Contents of /trunk/pkgtools/svn-cleanup-pkg-tree-sql.sh
Parent Directory | Revision Log
Revision 2108 -
(show annotations)
(download)
(as text)
Mon May 13 11:37:21 2013 UTC (11 years, 4 months ago) by niro
File MIME type: application/x-sh
File size: 7720 byte(s)
Mon May 13 11:37:21 2013 UTC (11 years, 4 months ago) by niro
File MIME type: application/x-sh
File size: 7720 byte(s)
added cleanup script with sqlite support with significant speed improvements
1 | #!/bin/bash |
2 | |
3 | # export default C locale |
4 | export LC_ALL=C |
5 | |
6 | MY_REPOS=( "branches/R11-stable" "trunk" ) |
7 | MY_ARCHES=( "x86_64" "i686" ) |
8 | MY_DISTRO_PKG_BASE="/var/www/htdocs/magellan/R11" |
9 | MY_DISTRO_HTTP_PKG_BASE="http://magellan-linux.de/magellan/R11" |
10 | MY_SSHLOGIN_BASE="tjoke@magellan-linux.de" |
11 | MY_MAGE_FEATURES=( "pkgdistrotag" ) |
12 | MY_MAGE_FEATURES_i686=( "srcpkgbuild" ) |
13 | |
14 | # decide wether to use http method or not |
15 | USE_HTTP=0 |
16 | if [[ $1 = --http ]] |
17 | then |
18 | USE_HTTP=1 |
19 | shift |
20 | fi |
21 | |
22 | MY_SVN_BASE="$1" |
23 | if [[ -z ${MY_SVN_BASE} ]] |
24 | then |
25 | MY_SVN_BASE="$(mktemp -d -t tree-cleanup.XXXXXXXXXX)" |
26 | fi |
27 | MY_SVN_OPTS="" |
28 | MY_SVN_ROOT="svn://cvs.magellan-linux.de" |
29 | |
30 | DBFILE="${MY_SVN_BASE}/pkgdb" |
31 | |
32 | sql() |
33 | { |
34 | local sqlcmd="$*" |
35 | [[ -z ${sqlcmd} ]] && die "no sqlcmd given." |
36 | |
37 | sqlite3 -nullvalue 'NULL' -list -separator '|' ${DBFILE} << EOF || die "error running '$@'" |
38 | ${sqlcmd}; |
39 | EOF |
40 | } |
41 | |
42 | # load_arch_values ARCH |
43 | load_arch_values() |
44 | { |
45 | local arch="$1" |
46 | |
47 | MAGE_FEATURES=( ${MY_MAGE_FEATURES[*]} ) |
48 | MAGE_FEATURES+=( $(eval echo \${MY_MAGE_FEATURES_${arch}[*]}) ) |
49 | export MAGE_FEATURES |
50 | |
51 | export ARCH="${arch}" |
52 | export CHOST="${arch}-pc-linux-gnu" |
53 | export PACKAGES_SERVER_PATH="packages/${arch}" |
54 | export PKGDIR="${MY_DISTRO_PKG_BASE}/packages/${arch}/" |
55 | export HTTP_PKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/${arch}/" |
56 | export SRCPKGDIR="${MY_DISTRO_PKG_BASE}/packages/src/" |
57 | export HTTP_SRCPKGDIR="${MY_DISTRO_HTTP_PKG_BASE}/packages/src/" |
58 | } |
59 | |
60 | # generate_pkg_file_list REPO |
61 | generate_pkg_file_list() |
62 | { |
63 | local repo="$1" |
64 | local smage |
65 | local arch |
66 | local _findlist |
67 | local subpackage |
68 | export SMAGESCRIPTSDIR="${repo}" |
69 | |
70 | _findlist=$(find ${repo} -name \*.${SMAGESUFFIX} | sort -u) |
71 | for arch in ${MY_ARCHES[*]} |
72 | do |
73 | load_arch_values ${arch} |
74 | load_mage_features |
75 | |
76 | for smage in ${_findlist} |
77 | do |
78 | echo "Processing ${smage}" |
79 | smagesource ${smage} &> /dev/null || die |
80 | |
81 | # honor split-packages |
82 | if [[ -n ${SPLIT_PACKAGES} ]] |
83 | then |
84 | local split_pkg_base="${PNAME}" |
85 | split_save_variables |
86 | for subpackage in ${SPLIT_PACKAGES} |
87 | do |
88 | export PNAME="${subpackage}" |
89 | split_info_${subpackage} |
90 | # fix PCATEGORIE -> PCAT |
91 | if [[ ! -z ${PCATEGORIE} ]] |
92 | then |
93 | PCAT="${PCATEGORIE}" |
94 | unset PCATEGORIE |
95 | fi |
96 | |
97 | # echo "DEBUG: split_package()" |
98 | # echo "DEBUG: PNAME='${PNAME}'" |
99 | # echo "DEBUG: PVER='${PVER}'" |
100 | # echo "DEBUG: PBUILD='${PBUILD}'" |
101 | # echo "DEBUG: SMAGESCRIPTSDIR='${SMAGESCRIPTSDIR}'" |
102 | # echo "DEBUG: DISTROTAG='${DISTROTAG}'" |
103 | # echo "DEBUG: print_distrotag()='$(print_distrotag)'" |
104 | # echo "DEBUG: ARCH='${ARCH}'" |
105 | # echo "DEBUG: CHOST='${CHOST}'" |
106 | # echo "DEBUG: PACKAGES_SERVER_PATH='${PACKAGES_SERVER_PATH}'" |
107 | # echo "DEBUG: PKGDIR='${PKGDIR}'" |
108 | # echo "DEBUG: SRCPKGDIR='${SRCPKGDIR}'" |
109 | |
110 | sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values( |
111 | '${PNAME}', |
112 | '${PVER}', |
113 | '${PBUILD}', |
114 | '${ARCH}', |
115 | '$(print_distrotag)', |
116 | '${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')" |
117 | |
118 | # restore smage environment |
119 | split_restore_variables |
120 | done |
121 | # unset all saved smage variables |
122 | split_unset_variables |
123 | unset split_pkg_base |
124 | else |
125 | sql "insert or ignore into pkgs(pname, pver, pbuild, arch, distrotag, filename ) values( |
126 | '${PNAME}', |
127 | '${PVER}', |
128 | '${PBUILD}', |
129 | '${ARCH}', |
130 | '$(print_distrotag)', |
131 | '${PNAME}${target}-${PVER}-${ARCH}$(print_distrotag)-${PBUILD}.${PKGSUFFIX}')" |
132 | fi |
133 | if mqueryfeature "srcpkgbuild" |
134 | then |
135 | sql "insert or ignore into srcpkgs(pname, pver, pbuild, filename ) values( |
136 | '${PNAME}', |
137 | '${PVER}', |
138 | '${PBUILD}', |
139 | '${PNAME}${target}-${PVER}-${PBUILD}.${SRCPKGSUFFIX}')" |
140 | fi |
141 | unset PNAME |
142 | unset PVER |
143 | unset PBUILD |
144 | unset SPLIT_PACKAGES |
145 | done |
146 | done |
147 | } |
148 | |
149 | # check_pkglist TABLE PKGFILE |
150 | check_pkglist() |
151 | { |
152 | local table="$1" |
153 | local pkg="$2" |
154 | |
155 | if [[ -z $(sql "select filename from ${table} where filename='${pkg}'") ]] |
156 | then |
157 | return 1 |
158 | fi |
159 | return 0 |
160 | } |
161 | |
162 | # checkout svn sources for mage and smage repos |
163 | export SVNROOT="${MY_SVN_ROOT}" |
164 | [[ ! -d ${MY_SVN_BASE} ]] && install -d ${MY_SVN_BASE} |
165 | for module in magellan-source/trunk/mage ${MY_REPOS[*]} |
166 | do |
167 | case ${module} in |
168 | magellan-source/*) repo="${module}"; codir=mage-src ;; |
169 | *) repo="smage/${module}"; codir="smage/${module}" ;; |
170 | esac |
171 | |
172 | if [ ! -d ${MY_SVN_BASE}/${module}/.svn ] |
173 | then |
174 | ( cd ${MY_SVN_BASE}; svn checkout ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} ) |
175 | else |
176 | ( cd ${MY_SVN_BASE}; svn update ${MY_SVN_OPTS} ${SVNROOT}/${repo} ${codir} ) |
177 | fi |
178 | |
179 | # exclude deprecated |
180 | if [ -d ${MY_SVN_BASE}/${codir}/deprecated ] |
181 | then |
182 | echo "deleting ${MY_SVN_BASE}/${codir}/deprecated" |
183 | rm -r ${MY_SVN_BASE}/${codir}/deprecated |
184 | fi |
185 | done |
186 | |
187 | # include functions from fresh mage checkout |
188 | source ${MY_SVN_BASE}/mage-src/usr/lib/mage/mage.rc.global.in |
189 | export MLIBDIR="${MY_SVN_BASE}/mage-src/usr/lib/mage" |
190 | source ${MLIBDIR}/mage4.functions.sh |
191 | source ${MLIBDIR}/smage2.functions.sh |
192 | |
193 | sql "create table IF NOT EXISTS pkgs(id integer primary key, |
194 | pname text, |
195 | pver text, |
196 | pbuild text, |
197 | arch text, |
198 | distrotag text, |
199 | filename text unique)" |
200 | sql "create table IF NOT EXISTS srcpkgs(id integer primary key, |
201 | pname text, |
202 | pver text, |
203 | pbuild text, |
204 | filename text unique)" |
205 | |
206 | for repo in ${MY_REPOS[*]} |
207 | do |
208 | generate_pkg_file_list "${MY_SVN_BASE}/smage/${repo}" |
209 | done |
210 | |
211 | sql "select filename from pkgs" |
212 | sql "select filename from srcpkgs" |
213 | |
214 | # initialize lists |
215 | if [[ ${USE_HTTP} = 1 ]] |
216 | then |
217 | :> ${MY_SVN_BASE}/pkglist |
218 | :> ${MY_SVN_BASE}/srcpkglist |
219 | fi |
220 | |
221 | for arch in ${MY_ARCHES[*]} |
222 | do |
223 | load_arch_values "${arch}" |
224 | load_mage_features |
225 | |
226 | if [[ ${USE_HTTP} = 1 ]] |
227 | then |
228 | _filelist=$(lynx -dump -listonly ${HTTP_PKGDIR} | grep ".${PKGSUFFIX}" | sed "s:.*${PACKAGES_SERVER_PATH}/\(.*.${PKGSUFFIX}\):\1:") |
229 | else |
230 | _filelist=$(find ${PKGDIR} -name \*.${PKGSUFFIX} -printf '%f\n' | sort) |
231 | fi |
232 | |
233 | if [[ ${USE_HTTP} = 1 ]] |
234 | then |
235 | echo "install -d ${PKGDIR}/_deprecated" >> ${MY_SVN_BASE}/pkglist |
236 | else |
237 | if [ ! -d ${PKGDIR}/_deprecated ] |
238 | then |
239 | install -d ${PKGDIR}/_deprecated || die "install ${PKGDIR}/${arch}/_deprecated" |
240 | fi |
241 | fi |
242 | for file in ${_filelist} |
243 | do |
244 | if check_pkglist pkgs "${file}" |
245 | then |
246 | echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}" |
247 | else |
248 | echo -e "${COLRED}[d] - ${file}${COLDEFAULT}" |
249 | if [[ ${USE_HTTP} = 1 ]] |
250 | then |
251 | echo "mv -v ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ " >> ${MY_SVN_BASE}/pkglist |
252 | else |
253 | mv ${PKGDIR}/${file} ${PKGDIR}/_deprecated/ || die "move ${PKGDIR}/${file} -> _deprecated" |
254 | fi |
255 | fi |
256 | done |
257 | if mqueryfeature "srcpkgbuild" |
258 | then |
259 | if [[ ${USE_HTTP} = 1 ]] |
260 | then |
261 | _filelist=$(lynx -dump -listonly ${HTTP_SRCPKGDIR} | grep ".${SRCPKGSUFFIX}" | sed "s:.*packages/src/\(.*.${SRCPKGSUFFIX}\):\1:") |
262 | else |
263 | _filelist=$(find ${SRCPKGDIR} -name \*.${SRCPKGSUFFIX} -printf '%f\n' | sort) |
264 | fi |
265 | |
266 | if [[ ${USE_HTTP} = 1 ]] |
267 | then |
268 | echo "install -d ${SRCPKGDIR}/_deprecated" >> ${MY_SVN_BASE}/srcpkglist |
269 | else |
270 | if [ ! -d ${SRCPKGDIR}/_deprecated ] |
271 | then |
272 | install -d ${SRCPKGDIR}/_deprecated || die "install ${SRCPKGDIR}/_deprecated" |
273 | fi |
274 | fi |
275 | for file in ${_filelist} |
276 | do |
277 | if check_pkglist srcpkgs "${file}" |
278 | then |
279 | echo -e "${COLGREEN}[k] - ${file}${COLDEFAULT}" |
280 | else |
281 | echo -e "${COLRED}[d] - ${file}${COLDEFAULT}" |
282 | if [[ ${USE_HTTP} = 1 ]] |
283 | then |
284 | echo "mv -v ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/" >> ${MY_SVN_BASE}/srcpkglist |
285 | else |
286 | mv ${SRCPKGDIR}/${file} ${SRCPKGDIR}/_deprecated/ || die "move ${SRCPKGDIR}/${file} -> _deprecated" |
287 | fi |
288 | fi |
289 | done |
290 | fi |
291 | done |
292 | |
293 | if [[ ${USE_HTTP} = 1 ]] |
294 | then |
295 | for list in pkglist srcpkglist |
296 | do |
297 | if [[ -f ${MY_SVN_BASE}/${list} ]] |
298 | then |
299 | scp ${MY_SVN_BASE}/${list} ${MY_SSHLOGIN_BASE}:~ |
300 | ssh ${MY_SSHLOGIN_BASE} bash ~/${list} |
301 | fi |
302 | done |
303 | fi |