diff options
63 files changed, 5601 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b25c15b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ @@ -0,0 +1,58 @@ + +WELCOME + +These are the installation instructions for LXRng, such as they are. + + +DEPENDENCIES + +(Package names for Ubunty Feisty in brackets, where applicable.) + +* Freetext index engine of your choice. Presently supported: + - Search::Xapian + +* Database and DBI modules of your choice. Presently supported: + - DBD::Pg [libdbd-pg-perl] + +* HTML/Web Perl modules + - CGI::Simple [libcgi-simple-perl] + - CGI::Ajax [libcgi-ajax-perl] + - HTML::Entities [libhtml-parser-perl] + - Template [libtemplate-perl] + +* "Exuberant ctags", runnable as ctags-exuberant somewhere in the + current $PATH. [exuberant-ctags] + +* (For generation of png icons from svg source: inkscape [inkscape]) + +* (For PDF generation: pdflatex [texlive texlive-latex-recommended + texlive-pdfetex texlive-fonts-extra lmodern]) + + +INSTALLATION + +* Create suitable database + $ createdb lxrng + +* Add www-data (or equivalent HTTP daemon user) as database user. No + special privileges should be afforded. + $ createuser www-data + +* Copy the lxrng.conf-dist to lxrng.conf and edit as desired. + +* Create database tables + $ lxr-db-admin <my-tree> --init + +* Cross reference your source repository + $ lxr-genxref <my-tree> + +* Copy the apache2-site.conf-dist to apache2-site.conf and adjust. + # ln -s <lxr-path>/apache2-site.conf /etc/apache2/sites-enabled/010-lxrng + # /etc/init.d/apache2 reload + (Or equivalent, depending on operating system (distribution) flavor.) + +* (Generate PNG icons) + $ make -C <lxr-path>/cgi-bin/gfx + +* Point web browser to configured web location. + diff --git a/apache2-site.conf-dist b/apache2-site.conf-dist new file mode 100644 index 0000000..0087eb5 --- /dev/null +++ b/apache2-site.conf-dist @@ -0,0 +1,8 @@ +Alias /lxr/ "@@LXRROOT@@/cgi-bin/" +<Directory "@@LXRROOT@@/cgi-bin/"> + Options None + AllowOverride All + Order deny,allow + Deny from none + Allow from all +</Directory> diff --git a/cgi-bin/.htaccess b/cgi-bin/.htaccess new file mode 100644 index 0000000..06615a1 --- /dev/null +++ b/cgi-bin/.htaccess @@ -0,0 +1,8 @@ +Options ExecCGI +<Files lxr> +SetHandler cgi-script +</Files> + +<Files prefs> +SetHandler cgi-script +</Files> diff --git a/cgi-bin/css/lxrng.css b/cgi-bin/css/lxrng.css new file mode 100644 index 0000000..9a47d80 --- /dev/null +++ b/cgi-bin/css/lxrng.css @@ -0,0 +1,278 @@ +/* http://devnull.tagsoup.com/fixed/horizontal.html */ + +span.lxr_l { + text-transform: uppercase; + margin-left: 5px; + font-weight: normal; +} + +span.lxr_x { + text-transform: uppercase; + font-weight: normal; +} + +span.lxr_r { + text-transform: uppercase; + font-weight: normal; + padding-right: 5px; + border-right: solid; + border-width: 1px; +} + +span.lxr_title { + float: left; +} + +span.lxr_menu { +/* float: right; */ +} +div.lxr_menu { + float: right; +} + +span.lxr_search { + font-weight: normal; + margin-left: 5px; + padding-left: 5px; + margin-right: 5px; +/* border-left: solid; + border-width: 1px; */ +} +span.lxr_prefs { + font-weight: normal; + margin-left: 5px; + padding-left: 5px; + margin-right: 5px; + border-left: solid; + border-width: 1px; +} + +span.lxr_version { + font-weight: normal; + padding-left: 5px; +/* border-left: solid; + border-width: 1px; */ +} + +body.full div.search_results { + background: #F0F0F0; + z-index: 3; + position: fixed; + right: 3px; + top: 35px; + width: 35%; + height: 70%; + border: solid; + border-width: 1px; + display: none; + overflow: auto; + padding: 3px; +} + +body.popup div.search_results { + background: #F0F0F0; + border: solid; + border-width: 1px; + padding: 3px; + margin: 1px; +} + +div.query_desc { + font-weight: bold; +} + +span.close-button { + float: right; + margin-top: 3px; + margin-right: 3px; +} + +a.line:before { + content: attr(id); +} + +a.line { + position: absolute; + top: auto; + left: 0px; + width: 4.5ex; + text-align: right; + background: #e0e0e0; + + border: solid; + border-width: 1px; + border-color: #000000; + margin-left: 3px; + padding-right: 5px; +} + +body { + margin-top: 3px; + margin-left: 3px; + margin-right: 3px; + overflow: scroll; +} + +pre { + margin-left: 6ex; +} + +div.headingtop { +} + +div.headingbottom { + clear: both; +} + +div.heading { + background: #F0F0F0; + margin-right: 0px; + margin-left: 0px; + font-weight: bold; + font-size: 120%; + border: solid; + border-width: 1px; + text-align: right; +} + +div.heading img { + vertical-align: middle; +} + +div.searchbox { + background: #F0F0F0; + border: solid; + margin-top: 3px; + border-width: 1px; + margin-right: 0px; + margin-left: 0px; + width: 30ex; + float: right; +} + + +button.print { + border: 0; + background: #F0F0F0; +} + + +table.query { + width: 100%; +} + +table.params { + width: 100%; +} + +table.directory { + width: 100%; +} + +table.directory td.name { + width: 30ex; +} + +table.directory td.size { + width: 10ex; + text-align: right; + padding-right: 1ex; +} + +table.directory td.time { + width: 30ex; +} + +div.footerbox { + left: 3px; + right: 3px; + bottom: 3px; + border: solid; + border-width: 1px; + background: #F0F0F0; +} + +div.footerfill { + height: 3px; +} + +div.content { + background: white; +} + + +span.comment { + font-weight: bold; + font-style: italic; +} + +span.string { + font-style: italic; + color: red; +} + +div.find { + padding: 3px; +} + +div.find div.find_input { + width: 100%; +} + +div.find div.find_code { + width: 33%; + float: left; + text-align: left; +} + +div.find div.find_text { + width: 33%; + float: left; + text-align: center; +} + +div.find div.find_file { + width: 33%; + float: right; + text-align: right; +} + +div.vars { + clear: both; + padding: 3px; +} + +div.vars div.var_title { + clear: both; + width: 50%; + float: left; + text-align: left; +} + +div.vars div.var_select { + width: 50%; + float: right; + text-align: right; +} + +div.vars div.do_update { + clear: both; + width: 50%; + float: left; + text-align: left; +} + +div.vars div.do_hide { + width: 50%; + float: right; + text-align: right; +} + +div.progress { + font-weight: bold; + font-style: italic; +} + +form { + display: inline; +} diff --git a/cgi-bin/gfx/Makefile b/cgi-bin/gfx/Makefile new file mode 100644 index 0000000..bef9f65 --- /dev/null +++ b/cgi-bin/gfx/Makefile @@ -0,0 +1,10 @@ +SVGFILES=$(wildcard *.svg) +PNGFILES=$(subst svg,png,${SVGFILES}) + +all: ${PNGFILES} + +clean: + rm -f ${PNGFILES} + +%.png: %.svg + inkscape -e $@ -w 16 -h 16 $< diff --git a/cgi-bin/gfx/close.png b/cgi-bin/gfx/close.png Binary files differnew file mode 100644 index 0000000..cd4178f --- /dev/null +++ b/cgi-bin/gfx/close.png diff --git a/cgi-bin/gfx/close.svg b/cgi-bin/gfx/close.svg new file mode 100644 index 0000000..2207f0f --- /dev/null +++ b/cgi-bin/gfx/close.svg @@ -0,0 +1,102 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://web.resource.org/cc/" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="32px" + height="32px" + id="svg2160" + sodipodi:version="0.32" + inkscape:version="0.45" + sodipodi:modified="TRUE" + inkscape:export-filename="foo" + inkscape:export-xdpi="67.5" + inkscape:export-ydpi="67.5"> + <defs + id="defs2162"> + <linearGradient + id="linearGradient3207"> + <stop + style="stop-color:#f19257;stop-opacity:0.31632653;" + offset="0" + id="stop3209" /> + <stop + style="stop-color:#e25213;stop-opacity:1;" + offset="1" + id="stop3211" /> + </linearGradient> + <linearGradient + id="linearGradient3185"> + <stop + style="stop-color:#ed2929;stop-opacity:1;" + offset="1" + id="stop3187" /> + </linearGradient> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient3207" + id="radialGradient3217" + cx="16" + cy="16" + fx="16" + fy="16" + r="15" + gradientUnits="userSpaceOnUse" /> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="11.197802" + inkscape:cx="16" + inkscape:cy="16" + inkscape:current-layer="layer1" + showgrid="true" + inkscape:grid-bbox="true" + inkscape:document-units="px" + inkscape:window-width="822" + inkscape:window-height="596" + inkscape:window-x="427" + inkscape:window-y="289" /> + <metadata + id="metadata2165"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> + <g + id="layer1" + inkscape:label="Layer 1" + inkscape:groupmode="layer"> + <rect + style="opacity:1;fill:url(#radialGradient3217);fill-opacity:1;fill-rule:nonzero;stroke:#000000;stroke-width:2;stroke-linecap:square;stroke-linejoin:round;stroke-miterlimit:16.69999886;stroke-dasharray:none;stroke-opacity:1" + id="rect2168" + width="30" + height="30" + x="1" + y="1" /> + <path + style="fill:none;fill-opacity:0.75;fill-rule:evenodd;stroke:#dfdfdf;stroke-width:5;stroke-linecap:butt;stroke-linejoin:round;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:none" + d="M 27,27 L 5,5" + id="path2170" /> + <path + style="fill:none;fill-opacity:0.75;fill-rule:evenodd;stroke:#dfdfdf;stroke-width:5;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="M 4.92738,27.193327 L 27.04318,5.1040233" + id="path3219" + sodipodi:nodetypes="cc" /> + </g> +</svg> diff --git a/cgi-bin/gfx/diff.png b/cgi-bin/gfx/diff.png Binary files differnew file mode 100644 index 0000000..73edeca --- /dev/null +++ b/cgi-bin/gfx/diff.png diff --git a/cgi-bin/gfx/diff.svg b/cgi-bin/gfx/diff.svg new file mode 100644 index 0000000..46290d3 --- /dev/null +++ b/cgi-bin/gfx/diff.svg @@ -0,0 +1,129 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://web.resource.org/cc/" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="32px" + height="32px" + id="svg2160" + sodipodi:version="0.32" + inkscape:version="0.45" + inkscape:export-filename="foo" + inkscape:export-xdpi="67.5" + inkscape:export-ydpi="67.5" + sodipodi:docbase="/home/argggh/privat/tshirt-art" + sodipodi:docname="diff.svg" + inkscape:output_extension="org.inkscape.output.svg.inkscape" + sodipodi:modified="true"> + <defs + id="defs2162"> + <linearGradient + id="linearGradient3207"> + <stop + style="stop-color:#e8ef1f;stop-opacity:0.3137255;" + offset="0" + id="stop3209" /> + <stop + style="stop-color:#e8ef1f;stop-opacity:1;" + offset="1" + id="stop3211" /> + </linearGradient> + <linearGradient + id="linearGradient3185"> + <stop + style="stop-color:#ed2929;stop-opacity:1;" + offset="1" + id="stop3187" /> + </linearGradient> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient3207" + id="radialGradient3217" + cx="16" + cy="16" + fx="16" + fy="16" + r="15" + gradientUnits="userSpaceOnUse" /> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient3207" + id="radialGradient3246" + gradientUnits="userSpaceOnUse" + cx="23.32336" + cy="10.578739" + fx="23.32336" + fy="10.578739" + r="15" + gradientTransform="matrix(-0.740049,-0.6725529,0.6904064,-0.7596943,32.018907,33.65448)" /> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient3207" + id="radialGradient3250" + gradientUnits="userSpaceOnUse" + gradientTransform="matrix(0.740049,-0.6725529,-0.6904064,-0.7596943,-1.8907e-2,45.65448)" + cx="23.32336" + cy="10.578739" + fx="23.32336" + fy="10.578739" + r="15" /> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="15.836083" + inkscape:cx="16.28893" + inkscape:cy="16.801809" + inkscape:current-layer="layer1" + showgrid="true" + inkscape:grid-bbox="false" + inkscape:document-units="px" + inkscape:window-width="822" + inkscape:window-height="1025" + inkscape:window-x="280" + inkscape:window-y="96" + objecttolerance="50" + guidetolerance="50" + inkscape:object-nodes="false" + inkscape:object-points="false" + inkscape:object-bbox="false" + inkscape:guide-bbox="false" + inkscape:grid-points="true" + inkscape:object-paths="false" /> + <metadata + id="metadata2165"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> + <g + id="layer1" + inkscape:label="Layer 1" + inkscape:groupmode="layer"> + <path + style="fill:url(#radialGradient3246);fill-opacity:1;fill-rule:nonzero;stroke:#000000;stroke-width:2;stroke-linecap:square;stroke-linejoin:round;stroke-miterlimit:16.69999886;stroke-opacity:1" + d="M 31,11 L 21,1 L 21,6 L 14,6 L 14,14 L 21,14 L 21,19 L 31,11 z " + id="rect2168" + sodipodi:nodetypes="cccccccc" /> + <path + style="fill:url(#radialGradient3250);fill-opacity:1;fill-rule:nonzero;stroke:#000000;stroke-width:2;stroke-linecap:square;stroke-linejoin:round;stroke-miterlimit:16.69999886;stroke-opacity:1" + d="M 1,21 L 11,12 L 11,17 L 18,17 L 18,25 L 11,25 L 11,30 L 1,21 z " + id="path3248" + sodipodi:nodetypes="cccccccc" /> + </g> +</svg> diff --git a/cgi-bin/gfx/left.png b/cgi-bin/gfx/left.png Binary files differnew file mode 100644 index 0000000..b01b78c --- /dev/null +++ b/cgi-bin/gfx/left.png diff --git a/cgi-bin/gfx/left.svg b/cgi-bin/gfx/left.svg new file mode 100644 index 0000000..4a3b1df --- /dev/null +++ b/cgi-bin/gfx/left.svg @@ -0,0 +1,102 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://web.resource.org/cc/" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="32px" + height="32px" + id="svg2160" + sodipodi:version="0.32" + inkscape:version="0.45" + inkscape:export-filename="foo" + inkscape:export-xdpi="67.5" + inkscape:export-ydpi="67.5" + sodipodi:docbase="/home/argggh/privat/tshirt-art" + sodipodi:docname="left.svg" + inkscape:output_extension="org.inkscape.output.svg.inkscape" + sodipodi:modified="true"> + <defs + id="defs2162"> + <linearGradient + id="linearGradient3207"> + <stop + style="stop-color:#288b0a;stop-opacity:0.3137255;" + offset="0" + id="stop3209" /> + <stop + style="stop-color:#298b0a;stop-opacity:1;" + offset="1" + id="stop3211" /> + </linearGradient> + <linearGradient + id="linearGradient3185"> + <stop + style="stop-color:#ed2929;stop-opacity:1;" + offset="1" + id="stop3187" /> + </linearGradient> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient3207" + id="radialGradient3217" + cx="16" + cy="16" + fx="16" + fy="16" + r="15" + gradientUnits="userSpaceOnUse" /> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="15.836083" + inkscape:cx="16" + inkscape:cy="16.801809" + inkscape:current-layer="layer1" + showgrid="true" + inkscape:grid-bbox="false" + inkscape:document-units="px" + inkscape:window-width="822" + inkscape:window-height="1025" + inkscape:window-x="280" + inkscape:window-y="96" + objecttolerance="50" + guidetolerance="50" + inkscape:object-nodes="false" + inkscape:object-points="false" + inkscape:object-bbox="false" + inkscape:guide-bbox="false" + inkscape:grid-points="true" + inkscape:object-paths="false" /> + <metadata + id="metadata2165"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> + <g + id="layer1" + inkscape:label="Layer 1" + inkscape:groupmode="layer"> + <path + style="fill:url(#radialGradient3217);fill-opacity:1;fill-rule:nonzero;stroke:#000000;stroke-width:2;stroke-linecap:square;stroke-linejoin:round;stroke-miterlimit:16.69999886;stroke-opacity:1" + d="M 1,16 L 20,1 L 20,9 L 31,9 L 31,23 L 20,23 L 20,31 L 1,16 z " + id="rect2168" + sodipodi:nodetypes="cccccccc" /> + </g> +</svg> diff --git a/cgi-bin/gfx/print.png b/cgi-bin/gfx/print.png Binary files differnew file mode 100644 index 0000000..4e56c0e --- /dev/null +++ b/cgi-bin/gfx/print.png diff --git a/cgi-bin/gfx/print.svg b/cgi-bin/gfx/print.svg new file mode 100644 index 0000000..b981609 --- /dev/null +++ b/cgi-bin/gfx/print.svg @@ -0,0 +1,149 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://web.resource.org/cc/" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="32px" + height="32px" + id="svg2160" + sodipodi:version="0.32" + inkscape:version="0.45" + inkscape:export-filename="foo" + inkscape:export-xdpi="67.5" + inkscape:export-ydpi="67.5" + sodipodi:docbase="/home/argggh/projects/lxrng/cgi-bin/gfx" + sodipodi:docname="print.svg" + inkscape:output_extension="org.inkscape.output.svg.inkscape" + sodipodi:modified="true"> + <defs + id="defs2162"> + <linearGradient + id="linearGradient3207"> + <stop + style="stop-color:#288b0a;stop-opacity:0.3137255;" + offset="0" + id="stop3209" /> + <stop + style="stop-color:#298b0a;stop-opacity:1;" + offset="1" + id="stop3211" /> + </linearGradient> + <linearGradient + id="linearGradient3185"> + <stop + style="stop-color:#ed2929;stop-opacity:1;" + offset="1" + id="stop3187" /> + </linearGradient> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="15.836083" + inkscape:cx="16" + inkscape:cy="16.801809" + inkscape:current-layer="layer1" + showgrid="true" + inkscape:grid-bbox="false" + inkscape:document-units="px" + inkscape:window-width="822" + inkscape:window-height="1025" + inkscape:window-x="605" + inkscape:window-y="34" + objecttolerance="50" + guidetolerance="50" + inkscape:object-nodes="false" + inkscape:object-points="false" + inkscape:object-bbox="false" + inkscape:guide-bbox="false" + inkscape:grid-points="true" + inkscape:object-paths="false" /> + <metadata + id="metadata2165"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> + <g + id="layer1" + inkscape:label="Layer 1" + inkscape:groupmode="layer" + style="display:inline"> + <path + style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:#000000;stroke-width:2;stroke-linecap:square;stroke-linejoin:round;stroke-miterlimit:16.69999886;stroke-opacity:1" + d="M 7,2 L 31,2 L 31,30 L 7,30 L 7,2 z " + id="rect2168" + sodipodi:nodetypes="ccccc" /> + <path + style="fill:none;fill-rule:evenodd;stroke:#979797;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="M 10,6 L 25,6" + id="path3134" /> + <path + style="fill:none;fill-rule:evenodd;stroke:#979797;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="M 10,10 L 18,10" + id="path3136" + sodipodi:nodetypes="cc" /> + <path + style="fill:none;fill-rule:evenodd;stroke:#979797;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="M 10,14 L 23,14" + id="path3138" + sodipodi:nodetypes="cc" /> + <path + style="fill:none;fill-rule:evenodd;stroke:#979797;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="M 10,18 L 15,18" + id="path3140" + sodipodi:nodetypes="cc" /> + <path + style="fill:none;fill-rule:evenodd;stroke:#979797;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="M 10,24 L 25,24" + id="path3142" /> + </g> + <g + inkscape:groupmode="layer" + id="layer2" + inkscape:label="PDF" + style="display:inline"> + <rect + style="fill:#ff0901;fill-opacity:1;stroke:#ff0901;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="rect3145" + width="24" + height="10" + x="3" + y="7" + ry="0" /> + <path + style="fill:none;fill-rule:evenodd;stroke:#ffffff;stroke-width:2;stroke-linecap:square;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="M 5,15 L 5,9 L 9,9 L 9,13 L 5,13" + id="path3147" + sodipodi:nodetypes="ccccc" /> + <path + style="fill:none;fill-rule:evenodd;stroke:#fefefe;stroke-width:2;stroke-linecap:square;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="M 13,9 L 13,15 L 16,15 L 17,14 L 17,10 L 16,9 L 13,9 z " + id="path3149" + sodipodi:nodetypes="ccccccc" /> + <path + style="fill:none;fill-rule:evenodd;stroke:#ffffff;stroke-width:2;stroke-linecap:square;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="M 21,15 L 21,9 L 25,9" + id="path3151" + sodipodi:nodetypes="ccc" /> + <path + style="fill:none;fill-rule:evenodd;stroke:#ffffff;stroke-width:2;stroke-linecap:square;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + d="M 21,13 L 23,13" + id="path3153" + sodipodi:nodetypes="cc" /> + </g> +</svg> diff --git a/cgi-bin/gfx/right.png b/cgi-bin/gfx/right.png Binary files differnew file mode 100644 index 0000000..439fe13 --- /dev/null +++ b/cgi-bin/gfx/right.png diff --git a/cgi-bin/gfx/right.svg b/cgi-bin/gfx/right.svg new file mode 100644 index 0000000..07e8941 --- /dev/null +++ b/cgi-bin/gfx/right.svg @@ -0,0 +1,102 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://web.resource.org/cc/" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="32px" + height="32px" + id="svg2160" + sodipodi:version="0.32" + inkscape:version="0.45" + inkscape:export-filename="foo" + inkscape:export-xdpi="67.5" + inkscape:export-ydpi="67.5" + sodipodi:docbase="/home/argggh/privat/tshirt-art" + sodipodi:docname="right.svg" + inkscape:output_extension="org.inkscape.output.svg.inkscape" + sodipodi:modified="true"> + <defs + id="defs2162"> + <linearGradient + id="linearGradient3207"> + <stop + style="stop-color:#288b0a;stop-opacity:0.3137255;" + offset="0" + id="stop3209" /> + <stop + style="stop-color:#298b0a;stop-opacity:1;" + offset="1" + id="stop3211" /> + </linearGradient> + <linearGradient + id="linearGradient3185"> + <stop + style="stop-color:#ed2929;stop-opacity:1;" + offset="1" + id="stop3187" /> + </linearGradient> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient3207" + id="radialGradient3217" + cx="16" + cy="16" + fx="16" + fy="16" + r="15" + gradientUnits="userSpaceOnUse" /> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="15.836083" + inkscape:cx="16.28893" + inkscape:cy="16.801809" + inkscape:current-layer="layer1" + showgrid="true" + inkscape:grid-bbox="false" + inkscape:document-units="px" + inkscape:window-width="822" + inkscape:window-height="1025" + inkscape:window-x="280" + inkscape:window-y="96" + objecttolerance="50" + guidetolerance="50" + inkscape:object-nodes="false" + inkscape:object-points="false" + inkscape:object-bbox="false" + inkscape:guide-bbox="false" + inkscape:grid-points="true" + inkscape:object-paths="false" /> + <metadata + id="metadata2165"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> + <g + id="layer1" + inkscape:label="Layer 1" + inkscape:groupmode="layer"> + <path + style="fill:url(#radialGradient3217);fill-opacity:1;fill-rule:nonzero;stroke:#000000;stroke-width:2;stroke-linecap:square;stroke-linejoin:round;stroke-miterlimit:16.69999886;stroke-opacity:1" + d="M 31,16 L 12,1 L 12,9 L 1,9 L 1,23 L 12,23 L 12,31 L 31,16 z " + id="rect2168" + sodipodi:nodetypes="cccccccc" /> + </g> +</svg> diff --git a/cgi-bin/gfx/rolldown.png b/cgi-bin/gfx/rolldown.png Binary files differnew file mode 100644 index 0000000..a2a15ca --- /dev/null +++ b/cgi-bin/gfx/rolldown.png diff --git a/cgi-bin/gfx/rolldown.svg b/cgi-bin/gfx/rolldown.svg new file mode 100644 index 0000000..85697fb --- /dev/null +++ b/cgi-bin/gfx/rolldown.svg @@ -0,0 +1,102 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://web.resource.org/cc/" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="32px" + height="32px" + id="svg2160" + sodipodi:version="0.32" + inkscape:version="0.45" + inkscape:export-filename="foo" + inkscape:export-xdpi="67.5" + inkscape:export-ydpi="67.5" + sodipodi:docbase="/home/argggh/privat/tshirt-art" + sodipodi:docname="rolldown.svg" + inkscape:output_extension="org.inkscape.output.svg.inkscape" + sodipodi:modified="true"> + <defs + id="defs2162"> + <linearGradient + id="linearGradient3207"> + <stop + style="stop-color:#0b718b;stop-opacity:0.3137255;" + offset="0" + id="stop3209" /> + <stop + style="stop-color:#0b718b;stop-opacity:1;" + offset="1" + id="stop3211" /> + </linearGradient> + <linearGradient + id="linearGradient3185"> + <stop + style="stop-color:#ed2929;stop-opacity:1;" + offset="1" + id="stop3187" /> + </linearGradient> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient3207" + id="radialGradient3217" + cx="16" + cy="16" + fx="16" + fy="16" + r="15" + gradientUnits="userSpaceOnUse" /> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="15.836083" + inkscape:cx="16" + inkscape:cy="16.801809" + inkscape:current-layer="layer1" + showgrid="true" + inkscape:grid-bbox="false" + inkscape:document-units="px" + inkscape:window-width="822" + inkscape:window-height="1025" + inkscape:window-x="280" + inkscape:window-y="96" + objecttolerance="50" + guidetolerance="50" + inkscape:object-nodes="false" + inkscape:object-points="false" + inkscape:object-bbox="false" + inkscape:guide-bbox="false" + inkscape:grid-points="true" + inkscape:object-paths="false" /> + <metadata + id="metadata2165"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + </cc:Work> + </rdf:RDF> + </metadata> + <g + id="layer1" + inkscape:label="Layer 1" + inkscape:groupmode="layer"> + <path + style="fill:url(#radialGradient3217);fill-opacity:1;fill-rule:nonzero;stroke:#000000;stroke-width:2;stroke-linecap:square;stroke-linejoin:round;stroke-miterlimit:16.69999886;stroke-opacity:1" + d="M 1,7 L 31,7 L 16.444972,27 L 1,7 z " + id="rect2168" + sodipodi:nodetypes="cccc" /> + </g> +</svg> diff --git a/cgi-bin/js/lxrng-funcs.js b/cgi-bin/js/lxrng-funcs.js new file mode 100644 index 0000000..d7c9be8 --- /dev/null +++ b/cgi-bin/js/lxrng-funcs.js @@ -0,0 +1,293 @@ +function popup_search(searchform) { + searchform = document.getElementById(searchform); + searchform.target = window.name + '-popup'; + searchform.navtarget.value = window.name; + window.open('', window.name + '-popup', + 'width=400,height=600,menubar=yes,status=yes,scrollbars=yes'); + return true; +} + +function popup_anchor() { + var anchor = this; + window.open('', window.name + '-popup', + 'width=400,height=600,menubar=yes,status=yes,scrollbars=yes'); + anchor.target = window.name + '-popup'; + + if (anchor.href.indexOf("navtarget=") >= 0) + return true; + + if (anchor.href.indexOf("?") >= 0) { + anchor.href = anchor.href + ';navtarget=' + window.name; + } + else { + anchor.href = anchor.href + '?navtarget=' + window.name; + } + return true; +} + +function navigate_here(searchform) { + searchform = document.getElementById(searchform); + searchform.target = window.name; + return true; +} + +function window_unique(serial) { + if (!window.name) + window.name = 'lxr-source-' + serial; +} + +function do_search(form) { + if (use_ajax_navigation) { + var res = document.getElementById('search_results'); + res.style.display = 'block'; + res.innerHTML = '<div class="progress">Searching...</div>'; + + pjx_search(['type__search', + 'search', 'v', 'tree__' + loaded_tree], + ['search_results']); + return false; + } + else if (use_popup_navigation) { + form.target = window.name + '-popup'; + form.navtarget.value = window.name; + reswin = window.open('', window.name + '-popup', + 'width=400,height=600,menubar=yes,status=yes,scrollbars=yes'); + } + return true; +} + +function hide_search() { + var res = document.getElementById('search_results'); + res.style.display = 'none'; + return false; +} + +var loaded_hash; +var loaded_tree; +var loaded_file; +var loaded_ver; +var loaded_line; + +var pending_tree; +var pending_file; +var pending_ver; +var pending_line; + +function ajax_nav() { + var file = this.href.replace(/^(http:.*?lxr\/[+]ajax\/|)/, ''); + // alert(loaded_file + ' - ' + file); + load_file(loaded_tree, file, loaded_ver, ''); + return false; +} + +function ajax_prefs() { + if (use_ajax_navigation) { + var full_path = location.href.split(/#/)[0]; + full_path = full_path + '/' + loaded_tree; + if (loaded_ver) { + full_path = full_path + '+' + loaded_ver; + } + full_path = full_path + '/+prefs?return=' + loaded_file.replace(/^\/?$/, '.'); + location = full_path; + return false; + } + else { + return true; + } +} + +var hash_check; +function check_hash_navigation() { + if (location.hash != loaded_hash) { + if (location.hash.replace(/\#L\d+$/, '') == + loaded_hash.replace(/\#L\d+$/, '')) + { + var l = location.hash.replace(/.*(\#L\d+)$/, '$1'); + var a = document.getElementById(l); + if (l && a) { + a.name = location.hash; + location.hash = a.name; + } + } + else { + // alert(location.hash + ' / ' + loaded_hash); + load_content(); + } + } + else { + hash_check = setTimeout('check_hash_navigation()', 50); + } +} + +function load_file(tree, file, ver, line) { + if (!use_ajax_navigation) { + return true; + } + + if (hash_check) { + clearTimeout(hash_check); + } + + var res = document.getElementById('content'); + + // TODO: check if file already loaded and perform only line + // location update. + res.innerHTML = '<div class="progress">Loading...</div>'; + pending_line = line; + pending_tree = tree; + pending_file = file; + if (ver) { + pending_ver = ver; + } + else { + pending_ver = ''; + } + + pjx_load_file(['tree__' + tree, 'file__' + file, 'v__' + ver, 'line__' + line], + [load_file_finalize]); + return false; +} + +function load_file_finalize(content) { + var res = document.getElementById('content'); + res.innerHTML = content; + var head = document.getElementById('current_path'); + head.innerHTML = '<a class=\"fref\" href=\".\">' + pending_tree + '</a>'; + var path_walked = ''; + var elems = pending_file.split(/\//); + for (var i=0; i<elems.length; i++) { + if (elems[i] != '') { + head.innerHTML = head.innerHTML + '/' + + '<a class=\"fref\" href=\"' + path_walked + elems[i] + + '\">' + elems[i] + '</a>'; + path_walked = path_walked + elems[i] + '/'; + } + } + document.title = 'LXR ' + pending_tree + '/' + pending_file; + + var full_path = pending_tree; + if (pending_ver) { + full_path = full_path + '+' + pending_ver; + } + full_path = full_path + '/' + pending_file.replace(/^\/?/, ''); + + var pre = document.getElementById('file_contents'); + if (pre && pre.className == 'partial') { + pjx_load_file(['tree__' + pending_tree, 'file__' + pending_file, + 'v__' + pending_ver, 'full__1'], + [load_file_finalize]); + } + + if (pending_line) { + var anchor = document.getElementById('L' + pending_line); + if (anchor) { + anchor.name = full_path + '#L' + pending_line; + location.hash = full_path + '#L' + pending_line; + } + else { + location.hash = full_path; + } + loaded_line = pending_line; + } + else { + location.hash = full_path; + loaded_line = 0; + } + loaded_hash = location.hash; + loaded_tree = pending_tree; + loaded_file = pending_file; + loaded_ver = pending_ver; + if (hash_check) { + clearTimeout(hash_check); + } + hash_check = setTimeout('check_hash_navigation()', 50); + + var i; + for (i = 0; i < document.links.length; i++) { + if (document.links[i].className == 'fref' || + document.links[i].className == 'line') + { + document.links[i].onclick = ajax_nav; + } + else if (document.links[i].className == 'sref' || + document.links[i].className == 'falt') + { + document.links[i].onclick = ajax_lookup_anchor; + } + + } +} + +function load_content() { + var tree = location.hash.split('/', 1); + tree = tree[0].split(/[+]/); + var ver = tree[1] || ''; + tree = tree[0].replace(/^#/, ''); + var file = location.hash.replace(/^[^\/]*\/?/, ''); + var line = file.replace(/.*\#L(\d+)/, '$1'); + file = file.replace(/\#L\d+$/, ''); + + load_file(tree, file, ver, line); + + pjx_releases(['tree__' + tree], + [load_content_finalize]); +} + +function load_content_finalize(content) { + var res = document.getElementById('ver_select'); + res.innerHTML = content; + var verlist = document.getElementById('ver_list'); + verlist.value = pending_ver; +} + +function update_version(verlist, base_url, tree, defversion, path) { + if (use_ajax_navigation) { + var file = location.hash.replace(/^[^\/]*\//, ''); + + load_file(loaded_tree, file, verlist.value, ''); + return false; + } + else { + var newurl = base_url.replace(/[^\/]*\/?$/, ''); + if (verlist.value == defversion) { + newurl = newurl + tree; + } + else { + newurl = newurl + tree + '+' + verlist.value; + } + newurl = newurl + '/' + path.replace(/^\//, ''); + document.location = newurl; + } +} + +function popup_prepare(serial) { + window_unique(serial); + var i; + for (i = 0; i < document.links.length; i++) { + if (document.links[i].className == 'sref' || + document.links[i].className == 'falt') + { + document.links[i].onclick = popup_anchor; + } + } +} + +function ajax_lookup_anchor(event, anchor) { + if (!use_ajax_navigation) + return true; + + if (!anchor) + anchor = this; + + lookup = anchor.href.replace(/^(http:.*?lxr\/[+]ajax\/|)/, ''); + var lvar = document.getElementById('ajax_lookup'); + lvar.value = lookup; + + var res = document.getElementById('search_results'); + res.style.display = 'block'; + res.innerHTML = '<div class="progress">Searching...</div>'; + + pjx_search(['ajax_lookup', 'v', 'tree__' + loaded_tree], + ['search_results']); + return false; +} diff --git a/cgi-bin/lxr b/cgi-bin/lxr new file mode 100755 index 0000000..cda9179 --- /dev/null +++ b/cgi-bin/lxr @@ -0,0 +1,659 @@ +#!/usr/bin/perl + +use strict; + +use FindBin; +use lib "$FindBin::Bin/../lib"; +use lib '/home/argggh/www/lxrng/_deps/share/perl/5.8.4'; + +use CGI::Carp qw(fatalsToBrowser); +use IO::Handle; + +use LXRng ROOT => "$FindBin::Bin/.."; +use LXRng::Context; +use LXRng::Lang qw(C); +use LXRng::Parse::Simple; +use LXRng::Markup::File; +use LXRng::Markup::Dir; +use Subst::Complex; + +use Template; +use Digest::SHA1 qw(sha1_hex); +use CGI::Ajax; +use CGI::Simple qw(-newstyle_urls); +use File::Temp qw(tempdir tempfile); +use POSIX qw(waitpid); + +use constant PDF_LINELEN => 95; +use constant PDF_CHARPTS => 6.6; + +use vars qw($has_gzip_io); +eval { require PerlIO::gzip; $has_gzip_io = 1; }; + + +# Return 1 if gzip compression of html is desired. + +sub do_compress_response { + my ($query) = @_; + + my @enc = split(",", $query->http('Accept-Encoding')); + return $has_gzip_io && grep { $_ eq 'gzip' } @enc; +} + + +# Progressive output of marked-up file. If the file in question +# exists in cache, and this is the initial load of an ajax-requested +# file, return only the lines the user wants to see (with a minimum of +# context) as a first approximation. + +sub print_markedup_file { + my ($context, $template, $node) = @_; + + autoflush STDOUT 1; + + if ($node->isa('LXRng::Repo::Directory')) { + my $markup = LXRng::Markup::Dir->new('context' => $context, + 'node' => $node); + $template->process('content_dir.tt2', + {'context' => $context, + 'dir_listing' => $markup->listing}) + or die $template->error(); + } + else { + # Grmble. We assume the identifiers to markup are identical + # from one version to another, but if the same revision of a + # file exists both in an indexed and un-indexed release, one + # of them will have its identifiers highlighted and the other + # not. So we can't share a cache slot across releases without + # adding some extra logic here. Bummer. + # TODO: Resolve by caching only accesses to releases that are + # is_indexed. + my $shaid = sha1_hex(join("\0", $node->name, $node->revision, + $context->release)); + my $cfile; + $cfile = $context->config->{'cache'}.'/'.$shaid + if exists $context->config->{'cache'}; + + if ($cfile and -e $cfile) { + open(my $cache, '<', $cfile); + + my $focus = $context->param('line') || 0; + $focus = 0 if $context->param('full'); + my $class = $focus ? 'partial' : 'full'; + print("<pre id=\"file_contents\" class=\"$class\">"); + while (<$cache>) { + next if $focus and $. < $focus - 5; + print($_); + last if $focus and $. > $focus + 70; + } + print("</pre>"); + close($cache); + } + else { + my $cache; + open($cache, '>', $cfile) if $cfile; + my $handle = $node->handle(); + my $lang = LXRng::Lang->new($node); + my $parse = LXRng::Parse::Simple->new($handle, 8, + @{$lang->parsespec}); + my $markup = LXRng::Markup::File->new('context' => $context); + my $subst = $lang->markuphandlers($context, $node, $markup); + + # Possible optimization: store cached file also as .gz, + # and pass that on if the client accepts gzip-encoded + # data. Saves us from compressing the cached file each + # time it's needed, but requires a bit of fiddling with + # perlio and the streams to get right. Also messes up + # partial transfers. + print("<pre id=\"file_contents\" class=\"full\">"); + while (1) { + my @frags = $markup->markupfile($subst, $parse); + last unless @frags; + print(@frags); + print($cache @frags) if $cache; + } + print("</pre>\n"); + } + } +} + +sub print_release_list { + my ($context, $template) = @_; + + $template->process('release_select.tt2', + {'context' => $context}) + or die $template->error(); +} + +sub source { + my ($context, $template, $query) = @_; + + my $pjx = CGI::Ajax->new('pjx_search' => 'lxr', + 'pjx_load_file' => 'lxr', + 'pjx_releases' => 'lxr'); + $pjx->js_encode_function('escape'); + + if ($context->prefs and $context->prefs->{'navmethod'} eq 'ajax') { + my $path = $query->path_info; + $path =~ s,^/[+ ],,; + if ($path ne '') { + $path =~ s,^/+,,; + print($query->redirect($query->url(-full => 1). + '#'.$path)); + } + else { + print($query->header(-type => 'text/html', + -charset => 'utf-8')); + + my $base = $query->url(-full => 1); + $template->process('main.tt2', + {'context' => $context, + 'base_url' => $base.'/+ajax/', + 'javascript' => $pjx->show_javascript(), + 'is_ajax' => 1}) + or die $template->error(); + } + return; + } + + my $gzip = do_compress_response($query); + + # history cookie + + print($query->header(-type => 'text/html', + -charset => 'utf-8', + $gzip ? (-content_encoding => 'gzip') : ())); + + binmode(\*STDOUT, ":gzip") if $gzip; + + my $ver = $context->release; + my $rep = $context->config->{'repository'}; + my $node = $rep->node($context->path, $ver); + + die "Node not found: ".$context->path." ($ver)" unless $node; + + my %template_args = ('context' => $context, + 'tree' => $context->tree, + 'node' => $node, + 'base_url' => $context->base_url, + 'javascript' => $pjx->show_javascript()); + + + if ($context->prefs and $context->prefs->{'navmethod'} eq 'popup') { + $template_args{'is_popup'} = 1; + $template_args{'popup_serial'} = int(rand(1000000)); + } + + if ($node->isa('LXRng::Repo::Directory')) { + my $markup = LXRng::Markup::Dir->new('context' => $context, + 'node' => $node); + $template->process('main.tt2', + {%template_args, + 'dir_listing' => $markup->listing, + 'is_dir' => 1}) + or die $template->error(); + } + else { + my $html = ''; + $template->process('main.tt2', + {%template_args, + 'file_content' => '<!--FILE_CONTENT-->', + 'is_dir' => 0}, + \$html) + or die $template->error(); + + # Template directives in processed template. Sigh. TT2 sadly + # can't do progressive rendering of its templates, so we cheat... + my ($pre, $post) = split('<!--FILE_CONTENT-->', $html); + print($pre); + print_markedup_file($context, $template, $node); + print($post); + } + + # TODO: This is potentially useful, in that it resets the stream + # to uncompressed mode. However, under Perl 5.8.8+PerlIO::gzip + # 0.18, this seems to truncate the stream. Not strictly needed + # for CGI, reexamine when adapting to mod_perl. + ## binmode(\*STDOUT, ":pop") if $gzip; +} + +#sub ident { +# my ($self) = @_; + +# my $index = $self->context->config->{'index'}; +# my $view = LXRng::View->new('context' => $self->context);; + +# my $ident = $self->context->value('ident'); +# my $target = $self->context->value('navtarget'); +# $target ||= 'source'; + +# my $rel_id = $index->release_id($self->tree, $self->context->value('v')); +# my ($symname, $symid, $ident, $refs) = +# $index->get_identifier_info($ident, $rel_id); + +# $$ident[1] = $LXRng::Lang::deftypes{$$ident[1]}; +# $$ident[5] &&= $LXRng::Lang::deftypes{$$ident[5]}; + +# return $view->identifier_info($symname, $symid, $ident, $refs, $target); +#} + + +# Perform various search operations. Return results as html suitable +# both as a response to an ajax request and inclusion in a more +# general html document. + +sub search { + my ($context, $template, $type, $find) = @_; + + my $ver = $context->release; + $find ||= $context->param('search'); + + my $index = $context->config->{'index'}; + my $rel_id = $index->release_id($context->tree, $ver); + my %template_args = ('context' => $context); + + $template_args{'navtarget'} = 'target='.$context->param('navtarget') + if $context->param('navtarget'); + + + if ($find =~ /\S/) { + if ($type eq 'file' or $type eq 'search') { + # $template_args{'file_res'} = {'query' => $find, + # 'files' => \@args,} + } + if ($type eq 'text' or $type eq 'search') { + my $hash = $context->config->{'search'}; + my ($total, $res) = $hash->search($rel_id, $find); + + $template_args{'text_res'} = {'query' => $find, + 'total' => $total, + 'files' => $res}; + } + if ($type eq 'code' or $type eq 'search') { + my $result = $index->symbols_by_name($context->tree, $ver, $find); + my @cooked = (map { $$_[1] = $LXRng::Lang::deftypes{$$_[1]}; + $$_[5] &&= $LXRng::Lang::deftypes{$$_[5]}; + $_ } + sort { $LXRng::Lang::defweight{$$a[1]} cmp + $LXRng::Lang::defweight{$$b[1]} } + @$result); + $template_args{'code_res'} = {'query' => $find, + 'idents' => \@cooked}; + } + if ($type eq 'ident') { + my ($symname, $symid, $ident, $refs) = + $index->get_identifier_info($find, $rel_id); + + $$ident[1] = $LXRng::Lang::deftypes{$$ident[1]}; + $$ident[5] &&= $LXRng::Lang::deftypes{$$ident[5]}; + + use Data::Dumper; + warn Dumper($symname, $symid, $ident, $refs); + $template_args{'ident_res'} = {'query' => $symname, + 'ident' => $ident, + 'refs' => $refs}; + } + if ($type eq 'ambig') { + my $rep = $context->config->{'repository'}; + my @args = grep { + $rep->node($_, $context->release) + } split(/\|/, $find); + $template_args{'ambig_res'} = {'query' => $find, + 'files' => \@args,} + } + } + else { + die "No query string given"; + } + my $html = ''; + $template_args{'tree'} = $context->tree; + $template->process('search_result.tt2', + \%template_args, + \$html) + or die $template->error(); + return $html; +} + + +# Display search results for plain and popup navigation methods. +# (Ajax methods call "search" directly.) + +sub search_result { + my ($context, $template, $query, $result) = @_; + + my %template_args = ('context' => $context, + 'tree' => $context->tree, + 'search_res' => $result, + 'base_url' => $context->base_url); + + my $gzip = do_compress_response($query); + + print($query->header(-type => 'text/html', + -charset => 'utf-8', + $gzip ? (-content_encoding => 'gzip') : ())); + + binmode(\*STDOUT, ":gzip") if $gzip; + + if ($context->prefs and $context->prefs->{'navmethod'} eq 'popup') { + $template->process('popup_main.tt2', + {%template_args, + 'is_popup' => 1}) + or die $template->error(); + } + else { + $template->process('main.tt2', + {%template_args, + 'file_content' => '', + 'is_dir' => 0}) + or die $template->error(); + } +} + + +# Callback to perform the ajax-available functions. + +sub handle_ajax_request { + my ($query, $context, $template) = @_; + my $gzip = do_compress_response($query); + + print($query->header(-type => 'text/html', + -charset => 'utf-8', + $gzip ? (-content_encoding => 'gzip') : ())); + + binmode(\*STDOUT, ":gzip") if $gzip; + + if ($context->param('fname') eq 'pjx_load_file') { + my $rep = $context->config->{'repository'}; + my $node = $rep->node($context->param('file'), $context->release); + print_markedup_file($context, $template, $node); + + } + elsif ($context->param('fname') eq 'pjx_search') { + if ($context->param('ajax_lookup') =~ + /^[+ ](code|ident|file|text|ambig)=(.*)/) + { + print(search($context, $template, $1, $2)); + } + else { + print(search($context, $template, 'search', + $context->param('search'))); + } + } + elsif ($context->param('fname') eq 'pjx_releases') { + print_release_list($context, $template); + } + + # binmode(\*STDOUT, ":pop") if $gzip; +} + + +# Stuff user preferences in cookie. + +sub handle_preferences { + my ($query, $context, $template) = @_; + + if ($context->param('resultloc')) { + my @prefs; + if ($context->param('resultloc') =~ /^(replace|popup|ajax)$/) { + push(@prefs, 'navmethod='.$1); + } + my $lxr_prefs = $query->cookie(-name => 'lxr_prefs', + -values => \@prefs, + -expires => '+1y'); + print($query->header(-type => 'text/html', + -charset => 'utf-8', + -cookie => $lxr_prefs)); + + my %template_args; + if (defined($context->param('return')) and $context->config) { + $template_args{'return'} = + $context->base_url.$query->param('return'); + } + else { + my $url = $query->url(-full => 1, -path => 1); + $url =~ s,/[+ ]prefs\b.*,/,; + $template_args{'return'} = $url; + } + + $template->process('prefs_set.tt2', + \%template_args) + or die $template->error(); + } + else { + print($query->header(-type => 'text/html', + -charset => 'utf-8')); + + $template->process('prefs.tt2', + {'return' => $query->param('return')}) + or die $template->error(); + } +} + + +# Generate pdf listing of given file. Much if the following lifted +# from the script "texify". Proof of concept, code quality could be +# better. + +sub generate_pdf { + my ($query, $context, $template, $path) = @_; + + my $tempdir = tempdir(CLEANUP => 1); + + my %tspecials = ( + '$' => '\$', '*' => "\$\\ast\$", + '&' => '\&', '%' => '\%', + '#' => '\#', '_' => '\_', + '^' => '\^{}', '{' => '\{', + '}' => '\}', '|' => "\$|\$", + '[' => '{[}', ']' => '{]}', + "'" => "{'}", "\"" => "\\string\"", + '~' => '\~{}', '<' => "\$<\$", + '>' => "\$>\$", "\\" => "\$\\backslash\$", + '-' => '\dash{}', + "\242" => '?', "\244" => '?', + "\245" => '?', "\246" => '?', + "\252" => "\$\252\$", "\254" => "\$\254\$", + "\255" => "\\dash{}", "\260" => "\$\260\$", + "\261" => "\$\261\$", "\262" => "\$\262\$", + "\263" => "\$\263\$", "\265" => "\$\265\$", + "\271" => "\$\271\$", "\272" => "\$\272\$", + "\327" => "\$\327\$", "\367" => "\$\367\$"); + + my $tspecials = join('', map { quotemeta($_) } keys(%tspecials)); + + my $ver = $context->release; + my $rep = $context->config->{'repository'}; + my $node = $rep->node($path, $ver); + + die "No such file" unless $node; + + my $handle = $node->handle(); + my $lang = LXRng::Lang->new($node); + my $parse = LXRng::Parse::Simple->new($handle, 8, + @{$lang->parsespec}); + my $res = $lang->reserved(); + my $resre; + if (%$res) { + $resre = '(?:(?<=[\s\W])|^)('. + join('|', map { my $c = $_; $c =~ s/\#/\\\#/g; quotemeta($c) } + sort { length($b) <=> length($a) } + keys %$res).')(?=$|[\s\W])'; + } + + my @lines; + my $row = 1; + my $col = 0; + my $line = '\\lxrln{1}'; + + while (1) { + my ($btype, $frag) = $parse->nextfrag; + + last unless defined $frag; + + $btype ||= 'code'; + my @parts = split(/(\n)/, $frag); + + while (@parts) { + my $part = shift(@parts); + my $align = 0; + my $cont = 0; + + if ($part eq "\n") { + push(@lines, $line); + + $col = 0; + $row++; + if ($row % 5 == 0) { + $line = "\\lxrln{$row}"; + } + else { + $line = ''; + } + next; + } + + if ($part =~ /^(.*? +)(.*)/) { + unshift(@parts, $2); + $part = $1; + $align = 1; + } + + $col += length($part); + + if ($col > PDF_LINELEN) { + unshift(@parts, + substr($part, PDF_LINELEN - $col, length($part), '')); + if ($part =~ s/([^\s_,\(\)\{\}\/\=\-\+\*\<\>\[\]\.]+)$//) { + if (length($1) < 20) { + unshift(@parts, $1); + } + else { + $part .= $1; + } + } + $align = 0; + $cont = 1; + } + + $part =~ s/([$tspecials])/$tspecials{$1}/ge; + + if ($btype eq 'code') { + $part =~ s/$resre/\\textbf{$1}/g if $resre; + } + elsif ($btype eq 'include') { + $part =~ s/$resre/\\textbf{$1}/ if $resre; + } + elsif ($btype eq 'comment') { + $part = '\textit{'.$part.'}'; + } + elsif ($btype eq 'string') { + $part = '\texttt{'.$part.'}'; + } + + # Common fixed-width "ascii-art" characters. + $part =~ s/(\$\\ast\$|=)/'\\makebox['.PDF_CHARPTS."pt][c]{$1}"/ge; + $line .= $part; + if ($align) { + $line = '\\makebox['.int($col * PDF_CHARPTS). + 'pt][l]{'.$line.'}'; + } + if ($cont) { + push(@lines, "$line\\raisebox{-2pt}{\\ArrowBoldRightStrobe}"); + $line = '\\raisebox{-2pt}{\\ArrowBoldDownRight} '; + $col = 3; + } + } + } + + if ($line ne '') { + push(@lines, $line); + } + else { + $row--; + } + + if (@lines and $row % 5 != 0) { + $lines[$#lines] =~ s/^/\\lxrln{$row}/; + } + + my $pathdesc = $context->tree."/$path ($ver)"; + $pathdesc =~ s/([$tspecials])/$tspecials{$1}/ge; + + my ($texh, $texname) = tempfile(DIR => $tempdir, SUFFIX => '.tex'); + + $template->process('print_pdf.tt2', + {'pathdesc' => $pathdesc, + 'lines' => \@lines}, + $texh) + or die $template->error(); + my $pid = fork(); + die $! unless defined($pid); + if ($pid == 0) { + close(STDOUT); + open(STDOUT, "> $texname.output"); + close(STDERR); + open(STDERR, ">&STDOUT"); + chdir($tempdir); + exec("pdflatex", "$texname"); + kill(9, $$); + } + waitpid($pid, 0); + my $pdfname = $texname; + $pdfname =~ s/[.]tex$/.pdf/; + if (-e $pdfname) { + open(my $pdfh, "< $pdfname") or die $!; + + print($query->header(-type => 'application/pdf', + -content_disposition => + "inline; filename=$path.pdf")); + my $buf = ''; + while (sysread($pdfh, $buf, 65536) > 0) { + print($buf); + } + close($pdfh); + } + elsif (-e "$texname.output") { + open(my $errh, "< $texname.output") or die $!; + my @err = <$errh>; + close($errh); + @err = splice(@err, -15) if @err > 15; + die "PDF generation failed: ".join("\n", @err); + } + else { + die "PDF generation failed"; + } +} + + +# Initial request dispatch. + +my $query = CGI::Simple->new(); +my $context = LXRng::Context->new('query' => $query); +my $template = Template->new({'INCLUDE_PATH' => $LXRng::ROOT.'/tmpl/'}); + + +if ($context->param('fname')) { + handle_ajax_request($query, $context, $template); +} +else { + if ($context->path =~ /^[+ ]prefs$/) { + handle_preferences($query, $context, $template); + } + elsif ($context->path =~ /^[+ ]print=(.*)/) { + generate_pdf($query, $context, $template, $1); + } + else { + if ($context->path =~ + /^[+ ](search|code|ident|file|text|ambig)(?:=(.*)|)/) + { + search_result($context, $template, $query, + search($context, $template, $1, $2)); + $context->path(''); + } + else { + source($context, $template, $query); + } + } +} + +1; diff --git a/lib/LXRng.pm b/lib/LXRng.pm new file mode 100644 index 0000000..11415cc --- /dev/null +++ b/lib/LXRng.pm @@ -0,0 +1,12 @@ +package LXRng; + +use strict; +use vars qw($ROOT); + +sub import { + my ($class, %args) = @_; + + $ROOT = $args{'ROOT'} if exists $args{'ROOT'}; +} + +1; diff --git a/lib/LXRng/Cached.pm b/lib/LXRng/Cached.pm new file mode 100644 index 0000000..f11a749 --- /dev/null +++ b/lib/LXRng/Cached.pm @@ -0,0 +1,63 @@ +package LXRng::Cached; + +use strict; +require Exporter; +use vars qw($memcached @ISA @EXPORT); +@ISA = qw(Exporter); +@EXPORT = qw(cached); + +BEGIN { + eval { require Cache::Memcached; + require Storable; + require Digest::SHA1; + }; + if ($@ eq '') { + $memcached = Cache::Memcached->new({ + 'servers' => ['127.0.0.1:11211']}); + $memcached = undef + unless ($memcached->set(':lxrng_caching' => 1)) + } +} + +# Caches result from block enclosed by cache { ... }. File/linenumber +# of the "cache" keyword is used as the caching key. If additional +# arguments are given after the sub to cache, they are used to further +# specify the caching key. Otherwise, the arguments supplied to the +# function containing the call to cached are used. + +sub cached(&;@); +*cached = \&DB::LXRng_Cached_cached; + +package DB; + +sub LXRng_Cached_cached(&;@) { + my ($func, @args) = @_; + if ($LXRng::Cached::memcached) { + my ($pkg, $file, $line) = caller(0); + my $params; + if (@args > 0) { + $params = Storable::freeze(\@args); + } + else { + my @caller = caller(1); + $params = Storable::freeze(\@DB::args); + } + my $key = ':lxrng:'. + Digest::SHA1::sha1_hex(join("\0", $file, $line, $params)); + my $val = $LXRng::Cached::memcached->get($key); + unless ($val) { + $val = [$func->()]; + $LXRng::Cached::memcached->set($key, $val); + warn "cache miss for $key"; + } + else { + warn "cache hit for $key"; + } + return @$val; + } + else { + return $func->(); + } +} + +1; diff --git a/lib/LXRng/Context.pm b/lib/LXRng/Context.pm new file mode 100644 index 0000000..46faa21 --- /dev/null +++ b/lib/LXRng/Context.pm @@ -0,0 +1,174 @@ +package LXRng::Context; + +use strict; +use LXRng; + +sub new { + my ($self, %args) = @_; + + $self = bless({}, $self); + + if ($args{'query'}) { + $$self{'req_url'} = $args{'query'}->url(); + + foreach my $p ($args{'query'}->param) { + $$self{'params'}{$p} = [$args{'query'}->param($p)]; + } + my @prefs = $args{'query'}->cookie('lxr_prefs'); + if (@prefs) { + $$self{'prefs'} = { + map { /^(.*?)(?:=(.*)|)$/; ($1 => $2) } @prefs }; + } + @$self{'tree', 'path'} = $args{'query'}->path_info =~ m,([^/]+)/*(.*),; + $$self{'tree'} = $args{'query'}->param('tree') + if $args{'query'}->param('tree'); + } + if ($args{'tree'}) { + $$self{'tree'} = $args{'tree'}; + } + + if ($$self{'tree'} =~ s/[+](.*)$//) { + $$self{'release'} = $1; + } + + if ($$self{'tree'}) { + my $tree = $$self{'tree'}; + my @config = $self->read_config(); + die("No config for tree $tree") + unless ref($config[0]) eq 'HASH' and exists($config[0]{$tree}); + + $$self{'config'} = $config[0]{$tree}; + } + + if (exists $$self{'params'}{'v'} and $$self{'params'}{'v'}) { + $$self{'release'} ||= $$self{'params'}{'v'}[0]; + delete($$self{'params'}{'v'}); + } + + if ($$self{'config'}) { + $$self{'release'} ||= $$self{'config'}{'ver_default'}; + } + + return $self; +} + +sub read_config { + my ($self) = @_; + + my $confpath = $LXRng::ROOT.'/lxrng.conf'; + + if (open(my $cfgfile, $confpath)) { + my @config = eval("use strict; use warnings;\n". + "#line 1 \"configuration file\"\n". + join("", <$cfgfile>)); + die($@) if $@; + + return @config; + } + else { + die("Couldn't open configuration file \"$confpath\"."); + } +} + +sub release { + my ($self, $value) = @_; + + $$self{'release'} = $value if @_ == 2; + return $$self{'release'}; +} + +sub default_release { + my ($self, $value) = @_; + + return $$self{'config'}{'ver_default'}; +} + +sub all_releases { + my ($self) = @_; + + return $$self{'config'}{'ver_list'}; +} + +sub param { + my ($self, $key) = @_; + my @res; + + @res = @{$$self{'params'}{$key}} if + exists $$self{'params'}{$key}; + + return wantarray ? @res : $res[0]; +} + +sub path { + my ($self, $value) = @_; + + $$self{'path'} = $value if @_ == 2; + return $$self{'path'}; +} + +sub tree { + my ($self) = @_; + + return $$self{'tree'}; +} + +sub vtree { + my ($self) = @_; + + if ($self->release ne $self->default_release) { + return $self->tree.'+'.$self->release; + } + else { + return $self->tree; + } +} + +sub path_elements { + my ($self) = @_; + + return [] if $self->path =~ /^[ +]/; + + my @path; + return [map { + push(@path, $_); { 'node' => $_, 'path' => join('', @path) } + } $self->path =~ m,([^/]+\/?),g]; +} + +sub config { + my ($self) = @_; + + return $$self{'config'}; +} + +sub prefs { + my ($self) = @_; + + return $$self{'prefs'}; +} + +sub base_url { + my ($self) = @_; + + my $base = $self->config->{'base_url'}; + unless ($base) { + $base = $$self{'req_url'}; + $base =~ s/lxr$//; + } + + $base =~ s,/+$,,; + $base .= '/lxr/'.$self->vtree.'/'; + $base =~ s,//+$,/,; + + return $base; +} + +sub args_url { + my ($self, %args) = @_; + + # Todo: escape + my $args = join(';', map { $_.'='.$args{$_} } keys %args); + $args = '?'.$args if $args; + return $args; +} + +1; diff --git a/lib/LXRng/Index.pm b/lib/LXRng/Index.pm new file mode 100644 index 0000000..e0d5794 --- /dev/null +++ b/lib/LXRng/Index.pm @@ -0,0 +1,11 @@ +package LXRng::Index; + +use strict; + +sub transaction(&@) { + my ($code, $index) = @_; + + $index->transaction($code); +} + +1; diff --git a/lib/LXRng/Index/DBI.pm b/lib/LXRng/Index/DBI.pm new file mode 100644 index 0000000..602eac8 --- /dev/null +++ b/lib/LXRng/Index/DBI.pm @@ -0,0 +1,430 @@ +package LXRng::Index::DBI; + +use strict; +use DBI; + +use base qw(LXRng::Index::Generic); + +sub transaction { + my ($self, $code) = @_; + if ($self->dbh->{AutoCommit}) { + $self->dbh->{AutoCommit} = 0; + $code->(); + $self->dbh->{AutoCommit} = 1; + } + else { + # If we're in a transaction already, don't return to + # AutoCommit state. + $code->(); + } + $self->dbh->commit(); +} + +sub _to_task { + my ($self, $rfile_id, $task) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_to_task_ins'} ||= + $dbh->prepare(qq{insert into ${pre}filestatus(id_rfile) + select ? where not exists + (select 1 from ${pre}filestatus + where id_rfile = ?)}); + $sth->execute($rfile_id, $rfile_id); + + $sth = $$self{'sth'}{'_to_task_upd'}{$task} ||= + $dbh->prepare(qq{update ${pre}filestatus set $task = 't' + where $task = 'f' and id_rfile = ?}); + return $sth->execute($rfile_id) > 0; +} + + +sub to_index { + my ($self, $rfile_id) = @_; + + return $self->_to_task($rfile_id, 'indexed'); +} + +sub to_reference { + my ($self, $rfile_id) = @_; + + return $self->_to_task($rfile_id, 'referenced'); +} + +sub to_hash { + my ($self, $rfile_id) = @_; + + return $self->_to_task($rfile_id, 'hashed'); +} + +sub _get_tree { + my ($self, $tree) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_get_tree'} ||= + $dbh->prepare(qq{select id from ${pre}trees where name = ?}); + my $id; + if ($sth->execute($tree) > 0) { + ($id) = $sth->fetchrow_array(); + } + $sth->finish(); + + return $id; +} + +sub _get_release { + my ($self, $tree_id, $release) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_get_release'} ||= + $dbh->prepare(qq{select id from ${pre}releases + where id_tree = ? and release_tag = ?}); + my $id; + if ($sth->execute($tree_id, $release) > 0) { + ($id) = $sth->fetchrow_array(); + } + $sth->finish(); + + return $id; +} + +sub _get_file { + my ($self, $path) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_get_file'} ||= + $dbh->prepare(qq{select id from ${pre}files where path = ?}); + my $id; + if ($sth->execute($path) > 0) { + ($id) = $sth->fetchrow_array(); + } + $sth->finish(); + + return $id; +} + +sub _get_rfile_by_release { + my ($self, $rel_id, $path) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_get_rfile_by_release'} ||= + $dbh->prepare(qq{select r.id + from ${pre}filereleases fr, ${pre}files f, + ${pre}revisions r + where fr.id_rfile = r.id and r.id_file = f.id + and fr.id_release = ? and f.path = ?}); + + my $id; + if ($sth->execute($rel_id, $path) > 0) { + ($id) = $sth->fetchrow_array(); + } + $sth->finish(); + + return $id; +} + +sub _get_symbol { + my ($self, $symbol) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_get_symbol'} ||= + $dbh->prepare(qq{select id from ${pre}symbols where name = ?}); + my $id; + if ($sth->execute($symbol) > 0) { + ($id) = $sth->fetchrow_array(); + } + $sth->finish(); + + return $id; +} + + +sub _add_include { + my ($self, $file_id, $inc_id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_add_include'} ||= + $dbh->prepare(qq{insert into ${pre}includes(id_rfile, id_include_path) + values (?, ?)}); + my $id; + $sth->execute($file_id, $inc_id); + + return 1; +} + +sub _includes_by_id { + my ($self, $file_id) = @_; + +} + +sub _symbol_by_id { + my ($self, $id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_symbol_by_id'} ||= + $dbh->prepare(qq{select * from ${pre}symbols + where id = ?}); + my @res; + if ($sth->execute($id) > 0) { + @res = $sth->fetchrow_array(); + } + $sth->finish(); + + return @res; +} + +sub _identifiers_by_name { + my ($self, $rel_id, $symbol) = @_; + + my $sym_id = $self->_get_symbol($symbol); + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_identifiers_by_name'} ||= + $dbh->prepare(qq{ + select i.id, i.type, f.path, i.line, s.name, c.type, c.id, + i.id_rfile + from ${pre}identifiers i + left outer join ${pre}identifiers c on i.context = c.id + left outer join ${pre}symbols s on c.id_symbol = s.id, + ${pre}files f, ${pre}filereleases r, ${pre}revisions v + where i.id_rfile = v.id and v.id = r.id_rfile + and r.id_release = ? and v.id_file = f.id + and i.id_symbol = ?}); + + $sth->execute($rel_id, $sym_id); + my $res = $sth->fetchall_arrayref(); + + use Data::Dumper; + foreach my $def (@$res) { +# warn Dumper($def); + $$def[7] = 42; +# my @files = $self->get_referring_files($rel_id, $$def[7]); +# warn Dumper(\@files); + } + + return $res; +} + +sub _symbols_by_file { + my ($self, $rfile_id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_symbols_by_file'} ||= + $dbh->prepare(qq{select distinct s.name + from ${pre}usage u, ${pre}symbols s + where id_rfile = ? and u.id_symbol = s.id}); + $sth->execute($rfile_id); + my %res; + while (my ($symname) = $sth->fetchrow_array()) { + $res{$symname} = 1; + } + + return \%res; +} + +sub _add_usage { + my ($self, $file_id, $line, $symbol_id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_add_usage'} ||= + $dbh->prepare(qq{insert into ${pre}usage(id_rfile, line, id_symbol) + values (?, ?, ?)}); + $sth->execute($file_id, $line, $symbol_id); + + return 1; +} + +sub _usage_by_file { + my ($self, $rfile_id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_usage_by_file'} ||= + $dbh->prepare(qq{select s.name, u.line + from ${pre}usage u, ${pre}symbols s + where id_rfile = ? and u.id_symbol = s.id}); + $sth->execute($rfile_id); + + die "Unimplemented"; +} + +sub _rfile_path_by_id { + my ($self, $rfile_id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_rfile_path_by_id'} ||= + $dbh->prepare(qq{select f.path from ${pre}files f, ${pre}revisions r + where f.id = r.id_file and r.id = ?}); + my $path; + if ($sth->execute($rfile_id) > 0) { + ($path) = $sth->fetchrow_array(); + } + $sth->finish(); + + return $path; +} + +sub _get_includes_by_file { + my ($self, $res, $rel_id, @rfile_ids) = @_; + + my $placeholders = join(', ', ('?') x @rfile_ids); + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $dbh->prepare(qq{select rf.id, f.path + from ${pre}revisions rf, + ${pre}filereleases v, + ${pre}includes i, + ${pre}revisions ri, + ${pre}files f + where rf.id = i.id_rfile + and rf.id_file = f.id + and rf.id = v.id_rfile + and v.id_release = ? + and i.id_include_path = ri.id_file + and ri.id in ($placeholders)}); + + + $sth->execute($rel_id, @rfile_ids); + my $files = $sth->fetchall_arrayref(); + $sth->finish(); + + my @recurse; + foreach my $r (@$files) { + push(@recurse, $$r[0]) unless exists($$res{$$r[0]}); + + $$res{$$r[0]} = $$r[1]; + } + + $self->_get_includes_by_file($res, $rel_id, @recurse) if @recurse; + + return 1; +} + +sub add_hashed_document { + my ($self, $rfile_id, $doc_id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'add_hashed_document'} ||= + $dbh->prepare(qq{insert into ${pre}hashed_documents(id_rfile, doc_id) + values (?, ?)}); + $sth->execute($rfile_id, $doc_id); + + return 1; +} + +sub get_hashed_document { + my ($self, $rfile_id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'get_hashed_document'} ||= + $dbh->prepare(qq{select doc_id from ${pre}hashed_documents + where id_rfile = ?}); + my $doc_id; + if ($sth->execute($rfile_id) > 0) { + ($doc_id) = $sth->fetchrow_array(); + } + $sth->finish(); + + return $doc_id; +} + +sub get_symbol_usage { + my ($self, $rel_id, $symid) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'get_symbol_usage'} ||= + $dbh->prepare(qq{ + select u.id_rfile, u.line + from ${pre}usage u, ${pre}filereleases fr + where u.id_symbol = ? + and u.id_rfile = fr.id_rfile and fr.id_release = ?}); + + $sth->execute($symid, $rel_id); + my $res = $sth->fetchall_arrayref(); + $sth->finish(); + + return $res; +} + +sub get_identifier_info { + my ($self, $ident, $rel_id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'get_identifier_info'} ||= + $dbh->prepare(qq{ + select s.name, s.id, + i.id, i.type, f.path, i.line, cs.name, c.type, c.id, + i.id_rfile + from ${pre}identifiers i + left outer join ${pre}identifiers c on i.context = c.id + left outer join ${pre}symbols cs on c.id_symbol = cs.id, + ${pre}symbols s, ${pre}revisions r, ${pre}files f + where i.id = ? and i.id_symbol = s.id + and i.id_rfile = r.id and r.id_file = f.id}); + +# select i.id_rfile, f.path, i.line, i.type, i.context, s.id, s.name +# from identifiers i, symbols s, revisions r, files f +# where i.id = ? and i.id_symbol = s.id +# and i.id_rfile = r.id and r.id_file = f.id}); + + unless ($sth->execute($ident) == 1) { + return undef; + } + + my ($symname, $symid, + $iid, $type, $path, $line, $cname, $ctype, $cid, $rfile_id) = + $sth->fetchrow_array(); + $sth->finish(); + + my $refs = {$rfile_id => $path}; + $self->get_referring_files($rel_id, $rfile_id, $refs); + my $usage = $self->get_symbol_usage($rel_id, $symid); + + my %reflines; + foreach my $u (@$usage) { + next unless $$refs{$$u[0]}; + $reflines{$$refs{$$u[0]}} ||= []; + push(@{$reflines{$$refs{$$u[0]}}}, $$u[1]); + } + + return ($symname, $symid, + [$iid, $type, $path, $line, $cname, $ctype, $cid, $rfile_id], + \%reflines); +} + +sub get_rfile_timestamp { + my ($self, $rfile_id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'get_rfile_timestamp'} ||= + $dbh->prepare(qq{ + select extract(epoch from last_modified_gmt)::integer, + last_modified_tz + from ${pre}revisions where id = ?}); + + unless ($sth->execute($rfile_id) == 1) { + return undef; + } + + my ($epoch, $tz) = $sth->fetchrow_array(); + $sth->finish(); + + return ($epoch, $tz); +} + +1; diff --git a/lib/LXRng/Index/Generic.pm b/lib/LXRng/Index/Generic.pm new file mode 100644 index 0000000..cab2513 --- /dev/null +++ b/lib/LXRng/Index/Generic.pm @@ -0,0 +1,172 @@ +package LXRng::Index::Generic; + +use strict; +use Memoize; + + +sub new { + my ($class, %args) = @_; + + memoize('tree_id'); + memoize('release_id'); + memoize('file_id'); + memoize('symbol_id'); + + return bless(\%args, $class); +} + +sub prefix { + my ($self) = @_; + if (exists $$self{'table_prefix'}) { + return $$self{'table_prefix'}.'_'; + } + else { + return ''; + } +} + +sub tree_id { + my ($self, $tree, $update) = @_; + + return $self->_get_tree($tree) || + ($update ? $self->_add_tree($tree) : undef); +} + +sub release_id { + my ($self, $tree, $release, $update) = @_; + + my $tree_id = $self->tree_id($tree, $update); + + return $self->_get_release($tree_id, $release) || + ($update ? $self->_add_release($tree_id, $release) : undef); +} + +sub file_id { + my ($self, $path, $update) = @_; + + return $self->_get_file($path) || + ($update ? $self->_add_file($path) : undef); +} + +sub rfile_id { + my ($self, $node, $update) = @_; + + my $path = $node->name; + my $revision = $node->revision; + + my $file_id = $self->file_id($path, $update); + return undef unless $file_id; + + my ($id, $old_stamp) = $self->_get_rfile($file_id, $revision); + return $id unless $update; + + if ($id) { + my ($new_stamp) = $node->time =~ /^(\d+)/; + if ($update and $old_stamp > $new_stamp) { + $self->_update_rfile_timestamp($id, $node->time); + } + } + else { + $id = $self->_add_rfile($file_id, $revision, $node->time); + } + return $id; +} + +sub symbol_id { + my ($self, $symbol, $update) = @_; + + return $self->_get_symbol($symbol) || + ($update ? $self->_add_symbol($symbol) : undef); +} + +sub add_filerelease { + my ($self, $tree, $release, $rfile_id) = @_; + + my $rel_id = $self->release_id($tree, $release, 1); + + $self->_add_filerelease($rfile_id, $rel_id); +} + +sub add_include { + my ($self, $file_id, $include_path) = @_; + + my $inc_id = $self->_get_file($include_path); + + return 0 unless $inc_id; + return $self->_add_include($file_id, $inc_id); +} + +sub includes_by_file { + my ($self, $tree, $release, $path) = @_; + + my $file_id = $self->file_id($tree, $release, $path); + + return $self->_includes_by_id($file_id); +} + +#sub add_symbol { +# my ($self, $file_id, $line, $symbol, $type, $ctx_id) = @_; +# return $self->_add_symbol($file_id, $line, $symbol, $type, $ctx_id); +#} + +sub add_ident { + my ($self, $rfile_id, $line, $symbol, $type, $ctx_id) = @_; + + my $sym_id = $self->symbol_id($symbol, 1); + + return $self->_add_ident($rfile_id, $line, $sym_id, $type, $ctx_id); +} + +sub symbol_by_id { + my ($self, $id) = @_; + + return $self->_symbol_by_id($id); +} + +sub symbols_by_name { + my ($self, $tree, $release, $symbol) = @_; + + my $rel_id = $self->release_id($tree, $release); +# return $cache_sym{$rel_id}{$symbol} if +# exists $cache_sym{$rel_id} and exists $cache_sym{$rel_id}{$symbol}; + return $self->_identifiers_by_name($rel_id, $symbol); +# $cache_sym{$rel_id}{$symbol} = $id; +} + +sub symbols_by_file { + my ($self, $tree, $release, $path) = @_; + + $path =~ s!^/!!; + my $rel_id = $self->_get_release($self->_get_tree($tree), $release); + my $rfile_id = $self->_get_rfile_by_release($rel_id, $path); + + return $self->_symbols_by_file($rfile_id); +} + +sub add_usage { + my ($self, $file_id, $line, $symbol) = @_; + + my $sym_id = $self->symbol_id($symbol, 1); + + return $self->_add_usage($file_id, $line, $sym_id); +} + +sub usage_by_file { + my ($self, $tree, $release, $path) = @_; + + my $rel_id = $self->_get_release($self->_get_tree($tree), $release); + my $rfile_id = $self->_get_rfile_by_release($rel_id, $path); + + return $self->_usage_by_file($rfile_id); +} + +sub get_referring_files { + my ($self, $rel_id, $rfile_id, $res) = @_; + + $res ||= {}; + $self->_get_includes_by_file($res, $rel_id, $rfile_id); + + return keys %$res; +} + +1; diff --git a/lib/LXRng/Index/Pg.pm b/lib/LXRng/Index/Pg.pm new file mode 100644 index 0000000..05fe3a0 --- /dev/null +++ b/lib/LXRng/Index/Pg.pm @@ -0,0 +1,417 @@ +package LXRng::Index::Pg; + +use strict; +use DBI; + +use base qw(LXRng::Index::DBI); + +sub dbh { + my ($self) = @_; + + $$self{'dbh'} ||= DBI->connect('dbi:Pg:'.$$self{'db_spec'}, + $$self{'db_user'}, $$self{'db_pass'}, + {AutoCommit => 1, + RaiseError => 1}) + or die($DBI::errstr); + + return $$self{'dbh'}; +} + +sub init_db { + my ($self) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + $dbh->{AutoCommit} = 0; + + $dbh->do(qq{create sequence ${pre}treenum}) or die($dbh->errstr); + $dbh->do(qq{create sequence ${pre}relnum}) or die($dbh->errstr); + $dbh->do(qq{create sequence ${pre}filenum cache 50}) or die($dbh->errstr); + $dbh->do(qq{create sequence ${pre}revnum cache 50}) or die($dbh->errstr); + $dbh->do(qq{create sequence ${pre}symnum cache 50}) or die($dbh->errstr); + $dbh->do(qq{create sequence ${pre}identnum cache 50}) or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}charsets + ( + id serial, + name varchar, + primary key (id) + ) + }) or die($dbh->errstr); + + $dbh->do(qq{insert into ${pre}charsets(name) values ('ascii')}) + or die($dbh->errstr); + $dbh->do(qq{insert into ${pre}charsets(name) values ('utf-8')}) + or die($dbh->errstr); + $dbh->do(qq{insert into ${pre}charsets(name) values ('iso8859-1')}) + or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}trees + ( + id int default nextval('${pre}treenum'), + name varchar, + primary key (id) + ) + }) or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}releases + ( + id int default nextval('${pre}relnum'), + id_tree int references ${pre}trees(id), + release_tag varchar, + is_indexed bool default 'f', + primary key (id) + ) + }) or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}files + ( + id int default nextval('${pre}filenum'), + path varchar, + primary key (id), + unique (path) + ) + }) or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}revisions + ( + id int default nextval('${pre}revnum'), + id_file int references ${pre}files(id), + revision varchar, + last_modified_gmt timestamp without time zone, -- GMT + last_modified_tz varchar(5), -- Optional TZ + body_charset int references ${pre}charsets(id), + primary key (id), + unique (id_file, revision) + ) + }) or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}filestatus + ( + id_rfile int references ${pre}revisions(id), + indexed bool default 'f', + referenced bool default 'f', + hashed bool default 'f', + primary key (id_rfile) + ) + }) or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}hashed_documents + ( + id_rfile int references ${pre}revisions(id), + doc_id int not null, + primary key (id_rfile) + ) + }) or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}filereleases + ( + id_rfile int references ${pre}revisions(id), + id_release int references ${pre}releases(id), + primary key (id_rfile, id_release) + ) + }) or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}includes + ( + id_rfile int references ${pre}revisions(id), + id_include_path int references ${pre}files(id) + ) + }) or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}symbols + ( + id int default nextval('${pre}symnum'), + name varchar, + primary key (id), + unique (name) + ) + }) or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}identifiers + ( + id int default nextval('${pre}identnum'), + id_symbol int references ${pre}symbols(id) deferrable, + id_rfile int references ${pre}revisions(id) deferrable, + line int, + type char(1), + context int references ${pre}identifiers(id) deferrable, + primary key (id) + ) + }) or die($dbh->errstr); + + $dbh->do(qq{ + create table ${pre}usage + ( + id_rfile int references ${pre}revisions(id) deferrable, + id_symbol int references ${pre}symbols(id) deferrable, + line int + ) + }) or die($dbh->errstr); + + $dbh->do(qq{alter table ${pre}usage alter column id_symbol set statistics 250}); + $dbh->do(qq{alter table ${pre}identifiers alter column id_symbol set statistics 250}); + + $dbh->do(qq{create index ${pre}symbol_idx1 on ${pre}symbols using btree (name)}) + or die($dbh->errstr); + $dbh->do(qq{create index ${pre}ident_idx1 on ${pre}identifiers using btree (id_symbol)}) + or die($dbh->errstr); + $dbh->do(qq{create index ${pre}ident_idx2 on ${pre}identifiers using btree (id_rfile)}) + or die($dbh->errstr); + $dbh->do(qq{create index ${pre}ident_idx3 on ${pre}identifiers using btree (id_symbol, id_rfile)}) + or die($dbh->errstr); + $dbh->do(qq{create index ${pre}usage_idx1 on ${pre}usage using btree (id_symbol)}) + or die($dbh->errstr); + $dbh->do(qq{create index ${pre}usage_idx2 on ${pre}usage using btree (id_rfile)}) + or die($dbh->errstr); + $dbh->do(qq{create index ${pre}include_idx1 on ${pre}includes using btree (id_rfile)}) + or die($dbh->errstr); + $dbh->do(qq{create index ${pre}file_idx1 on ${pre}files using btree (path)}) + or die($dbh->errstr); + + $dbh->do(qq{grant select on ${pre}charsets to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}trees to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}releases to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}files to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}releases to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}filereleases to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}filestatus to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}hashed_documents to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}includes to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}symbols to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}identifiers to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}usage to public}) or die($dbh->errstr); + $dbh->do(qq{grant select on ${pre}revisions to public}) or die($dbh->errstr); + + $dbh->commit(); + $dbh->{AutoCommit} = 0; +} + +sub drop_db { + my ($self) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + local($dbh->{RaiseError}) = 0; + + $dbh->do(qq{drop index ${pre}symbol_idx1}); + $dbh->do(qq{drop index ${pre}ident_idx1}); + $dbh->do(qq{drop index ${pre}ident_idx2}); + $dbh->do(qq{drop index ${pre}usage_idx1}); + $dbh->do(qq{drop index ${pre}usage_idx2}); + $dbh->do(qq{drop index ${pre}include_idx1}); + $dbh->do(qq{drop index ${pre}file_idx1}); + + $dbh->do(qq{drop table ${pre}usage}); + $dbh->do(qq{drop table ${pre}identifiers}); + $dbh->do(qq{drop table ${pre}symbols}); + $dbh->do(qq{drop table ${pre}includes}); + $dbh->do(qq{drop table ${pre}filereleases}); + $dbh->do(qq{drop table ${pre}hashed_documents}); + $dbh->do(qq{drop table ${pre}filestatus}); + $dbh->do(qq{drop table ${pre}revisions}); + $dbh->do(qq{drop table ${pre}files}); + $dbh->do(qq{drop table ${pre}releases}); + $dbh->do(qq{drop table ${pre}trees}); + $dbh->do(qq{drop table ${pre}charsets}); + + $dbh->do(qq{drop sequence ${pre}treenum}); + $dbh->do(qq{drop sequence ${pre}relnum}); + $dbh->do(qq{drop sequence ${pre}filenum}); + $dbh->do(qq{drop sequence ${pre}revnum}); + $dbh->do(qq{drop sequence ${pre}symnum}); + $dbh->do(qq{drop sequence ${pre}identnum}); +} + +sub _add_tree { + my ($self, $tree) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_add_tree_ins'} ||= + $dbh->prepare(qq{insert into ${pre}trees(name) values (?)}); + $sth->execute($tree); + + $sth = $$self{'sth'}{'_add_tree_insid'} ||= + $dbh->prepare(qq{select currval('${pre}treenum')}); + $sth->execute(); + my ($id) = $sth->fetchrow_array(); + $sth->finish(); + + return $id; +} + +sub _add_release { + my ($self, $tree_id, $release) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_add_relase_ins'} ||= + $dbh->prepare(qq{insert into ${pre}releases(id_tree, release_tag) + values (?, ?)}); + $sth->execute($tree_id, $release); + + $sth = $$self{'sth'}{'_add_release_insid'} ||= + $dbh->prepare(qq{select currval('${pre}relnum')}); + $sth->execute(); + my ($id) = $sth->fetchrow_array(); + $sth->finish(); + + return $id; +} + +sub _add_file { + my ($self, $path) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_add_file_ins'} ||= + $dbh->prepare(qq{insert into ${pre}files(path) values (?)}); + $sth->execute($path); + + $sth = $$self{'sth'}{'_add_file_insid'} ||= + $dbh->prepare(qq{select currval('${pre}filenum')}); + $sth->execute(); + my ($id) = $sth->fetchrow_array(); + $sth->finish(); + + return $id; +} + +sub _get_rfile { + my ($self, $file_id, $revision) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_get_rfile'} ||= + $dbh->prepare(qq{select id, extract('epoch' from last_modified_gmt + at time zone 'UTC') + from ${pre}revisions + where id_file = ? and revision = ?}); + my ($id, $gmt); + if ($sth->execute($file_id, $revision) > 0) { + ($id, $gmt) = $sth->fetchrow_array(); + } + $sth->finish(); + + return ($id, $gmt); +} + +sub _add_rfile { + my ($self, $file_id, $revision, $time) = @_; + + my ($epoch, $zone) = $time =~ /^(\d+)(?: ([-+]\d\d\d\d)|)$/; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_add_rfile_ins'} ||= + $dbh->prepare(qq{ + insert into ${pre}revisions(id_file, revision, + last_modified_gmt, + last_modified_tz) + values (?, ?, + timestamp 'epoch' + ? * interval '1 second', ?)}); + $sth->execute($file_id, $revision, $epoch, $zone) + or die($dbh->errstr); + + $sth = $$self{'sth'}{'_add_rfile_insid'} ||= + $dbh->prepare(qq{select currval('${pre}revnum')}); + $sth->execute(); + my ($id) = $sth->fetchrow_array(); + $sth->finish(); + + return $id; +} + +sub _update_rfile_timestamp { + my ($self, $rfile_id, $time) = @_; + + my ($epoch, $zone) = $time =~ /^(\d+)(?: ([-+]\d\d\d\d)|)$/; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_update_rfile_timestamp'} ||= + $dbh->prepare(qq{ + update ${pre}revisions set + last_modified_gmt = timestamp 'epoch' + ? * interval '1 second', + last_modified_tz = ? + where id = ?}); + + $sth->execute($epoch, $zone, $rfile_id) + or die($dbh->errstr); + $sth->finish(); +} + +sub _add_filerelease { + my ($self, $rfile_id, $rel_id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_add_filerelease'} ||= + $dbh->prepare(qq{insert into ${pre}filereleases(id_rfile, id_release) + select ?, ? where not exists + (select 1 from ${pre}filereleases + where id_rfile = ? and id_release = ?)}); + $sth->execute($rfile_id, $rel_id, $rfile_id, $rel_id); +} + +sub _add_symbol { + my ($self, $symbol) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_add_symbol_ins'} ||= + $dbh->prepare(qq{insert into ${pre}symbols(name) values (?)}); + $sth->execute($symbol); + + $sth = $$self{'sth'}{'_add_symbol_insid'} ||= + $dbh->prepare(qq{select currval('${pre}symnum')}); + $sth->execute(); + my ($id) = $sth->fetchrow_array(); + $sth->finish(); + + return $id; +} + +sub _add_ident { + my ($self, $rfile_id, $line, $sym_id, $type, $ctx_id) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_add_ident_ins'} ||= + $dbh->prepare(qq{insert into ${pre}identifiers + (id_rfile, line, id_symbol, type, context) + values (?, ?, ?, ?, ?)}); + $sth->execute($rfile_id, $line, $sym_id, $type, $ctx_id); + + $sth = $$self{'sth'}{'_add_ident_insid'} ||= + $dbh->prepare(qq{select currval('${pre}identnum')}); + $sth->execute(); + my ($id) = $sth->fetchrow_array(); + $sth->finish(); + + return $id; +} + +sub DESTROY { + my ($self) = @_; + + if ($$self{'dbh'}) { + $$self{'dbh'}->rollback(); + $$self{'dbh'}->disconnect(); + delete($$self{'dbh'}); + } +} + +1; diff --git a/lib/LXRng/Index/PgBatch.pm b/lib/LXRng/Index/PgBatch.pm new file mode 100644 index 0000000..8f8844c --- /dev/null +++ b/lib/LXRng/Index/PgBatch.pm @@ -0,0 +1,217 @@ +package LXRng::Index::PgBatch; + +# Specialized subclass of LXRng::Index::Pg for doing parallelized +# batched inserts into database. Higher performance (and higher +# complexity). + +use strict; +use DBI; +use POSIX qw(:sys_wait_h); + +use base qw(LXRng::Index::Pg); + + +sub transaction { + my ($self, $code) = @_; + + if ($self->dbh->{AutoCommit}) { + $self->dbh->{AutoCommit} = 0; + $self->dbh->do(q(set constraints all deferred)); + $code->(); + $self->flush(); + $self->dbh->commit(); + $self->dbh->{AutoCommit} = 1; + + # At end of outermost transaction, wait for outstanding flushes + $self->_flush_wait(); + } + else { + # If we're in a transaction already, don't return to + # AutoCommit state. + $code->(); + + # Only occasional synchronization if we're inside another + # transaction. + if ($self->{'writes'}++ % 997 == 0) { + $self->flush(); + $self->dbh->commit(); + } + } +} + +sub new { + my ($class, @args) = @_; + + my $self = $class->SUPER::new(@args); + $$self{'writes'} = 0; + + return $self; +} + +sub flush { + my ($self) = @_; + + return unless exists($$self{'cache'}); + + $self->_flush_wait(); + + my $pre = $self->prefix; + $self->dbh->commit() unless $self->dbh->{AutoCommit}; + my $pid = fork(); + die("fork failed: $!") unless defined($pid); + if ($pid == 0) { + $SIG{'INT'} = 'IGNORE'; + $SIG{'QUIT'} = 'IGNORE'; + $SIG{'TERM'} = 'IGNORE'; + + my $i = 0; + $$self{'dbh'} = undef; + foreach my $table (qw(symbols identifiers usage)) { + if (exists($$self{'cache'}{$table})) { + $self->dbh->do(qq{copy $pre$table from stdin}); + foreach my $l (@{$$self{'cache'}{$table}}) { + $i++; + $self->dbh->pg_putline($l); + } + $self->dbh->pg_endcopy; + } + } + $self->dbh->commit() unless $self->dbh->{AutoCommit}; + $self->dbh->do(q(analyze)) if $i > 100000; + $self->dbh->disconnect(); + warn "\n*** index: flushed $i rows\n"; + kill(9, $$); + } + $$self{'flush_pid'} = $pid; + delete($$self{'cache'}); + warn "\n*** index: flushing in background\n"; +} + +sub _flush_wait { + my ($self) = @_; + + return unless $$self{'flush_pid'}; + waitpid($$self{'flush_pid'}, WNOHANG); # Reap zombies + return unless kill(0, $$self{'flush_pid'}); + + warn "\n*** index: waiting for running flush to complete...\n"; + $self->dbh->commit() unless $self->dbh->{AutoCommit}; + waitpid($$self{'flush_pid'}, 0); +} + +sub _cache { + my ($self, $name) = @_; + + $$self{'cache'}{$name} ||= []; + return $$self{'cache'}{$name}; +} + +sub _cached_seqno { + my ($self, $seqname) = @_; + + unless (exists($$self{'cached_seqno'}{$seqname}) and + $$self{'cached_seqno'}{$seqname}{'min'} <= + $$self{'cached_seqno'}{$seqname}{'max'}) + { + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_cached_seqno'}{$seqname} ||= + $dbh->prepare(qq{select setval('$pre$seqname', + nextval('$pre$seqname')+1000)}); + $sth->execute(); + my ($id) = $sth->fetchrow_array(); + $sth->finish(); + $$self{'cached_seqno'}{$seqname}{'min'} = $id-1000; + $$self{'cached_seqno'}{$seqname}{'max'} = $id; + } + + return $$self{'cached_seqno'}{$seqname}{'min'}++; +} + +sub _prime_symbol_cache { + my ($self) = @_; + + my $dbh = $self->dbh; + my $pre = $self->prefix; + my $sth = $$self{'sth'}{'_prime_symbol_cache'} ||= + $dbh->prepare(qq{select name, id from ${pre}symbols}); + $sth->execute(); + my %cache; + while (my ($name, $id) = $sth->fetchrow_array()) { + $cache{$name} = $id; + } + $sth->finish; + + $$self{'__symbol_cache'} = \%cache; +} + +sub _add_usage { + my ($self, $file_id, $line, $symbol_id) = @_; + + push(@{$self->_cache('usage')}, "$file_id\t$symbol_id\t$line\n"); + + return 1; +} + +sub _add_symbol { + my ($self, $symbol) = @_; + + my $id = $self->_cached_seqno('symnum'); + push(@{$self->_cache('symbols')}, "$id\t$symbol\n"); + + $self->_prime_symbol_cache() + unless exists $$self{'__symbol_cache'}; + + $$self{'__symbol_cache'}{$symbol} = $id; + + return $id; +} + +sub _add_ident { + my ($self, $rfile_id, $line, $sym_id, $type, $ctx_id) = @_; + + $ctx_id = '\\N' unless defined($ctx_id); + + my $id = $self->_cached_seqno('identnum'); + push(@{$self->_cache('identifiers')}, join("\t", $id, $sym_id, + $rfile_id, $line, $type, + $ctx_id)."\n"); + + return $id; +} + +my $_get_symbol_usage = 0; +sub _get_symbol { + my ($self, $symbol) = @_; + + unless (exists($$self{'__symbol_cache'})) { + # Only prime the cache once it's clear that we're likely to + # hit it a significant number of times. + return $self->SUPER::_get_symbol($symbol) if + $_get_symbol_usage++ < 100; + + $self->_prime_symbol_cache(); + } + + return $$self{'__symbol_cache'}{$symbol} if + exists $$self{'__symbol_cache'}{$symbol}; + + return undef; +} + +sub DESTROY { + my ($self) = @_; + + if ($$self{'writes'} > 0) { + $self->flush(); + $self->_flush_wait(); + } + + if ($$self{'dbh'}) { + $$self{'dbh'}->rollback() unless $$self{'dbh'}{'AutoCommit'}; + $$self{'dbh'}->disconnect(); + delete($$self{'dbh'}); + } +} + +1; diff --git a/lib/LXRng/Lang.pm b/lib/LXRng/Lang.pm new file mode 100644 index 0000000..7e6c278 --- /dev/null +++ b/lib/LXRng/Lang.pm @@ -0,0 +1,56 @@ +package LXRng::Lang; + +use strict; +use vars qw(@languages %deftypes %defweight); + + +%deftypes = + ( + 'c' => 'class', + 'd' => 'macro (un)definition', + 'e' => 'enumerator', + 'f' => 'function', + 'g' => 'enumeration name', + 'm' => 'class, struct, or union member', + 'n' => 'namespace', + 'p' => 'function prototype or declaration', + 's' => 'structure', + 't' => 'typedef', + 'u' => 'union', + 'v' => 'variable', + 'l' => 'local variable', + 'x' => 'extern or forward variable declaration', + 'i' => 'interface' + ); + +%defweight = do { my $i = 0; + map { $_ => $i++ } + qw(c f i n s t u p x v d e g m l) }; + + +sub import { + my ($self, @langs) = @_; + + push(@langs, 'Undefined'); + foreach my $l (@langs) { + eval "require LXRng::Lang::$l"; + die $@ if $@; + push(@languages, "LXRng::Lang::$l"); + } +} + +sub new { + my ($self, $file) = @_; + + my $pathname = $file->name(); + + foreach my $l (@languages) { + if ($pathname =~ $l->pathexp) { + return $l; + } + } + + die "No language found for $pathname"; +} + +1; diff --git a/lib/LXRng/Lang/C.pm b/lib/LXRng/Lang/C.pm new file mode 100644 index 0000000..c88f424 --- /dev/null +++ b/lib/LXRng/Lang/C.pm @@ -0,0 +1,137 @@ +package LXRng::Lang::C; + +use strict; +use Subst::Complex; + +use base qw(LXRng::Lang::Generic); + + +sub doindex { + return 1; +} + +sub ctagslangname { + return 'c'; +} + +sub ctagsopts { + return ('--c-types=+lpx'); +} + +sub pathexp { + return qr/\.[ch]$/; +} + +my $_identifier_re = qr( + (?m:^|(?<=[^a-zA-Z0-9_\#])) # Non-symbol chars. + (_*[a-zA-Z][a-zA-Z0-9_]*) # The symbol. + \b + )x; + +sub identifier_re { + return $_identifier_re; +} + +my $_reserved ||= { map { $_ => 1 } + qw(asm auto break case char continue default do + double else enum extern float for fortran goto + if int long register return short signed sizeof + static struct switch typedef union unsigned + void volatile while + #define #else #endif #if #ifdef #ifndef #include + #undef)}; + +sub reserved { + return $_reserved; +} + +sub parsespec { + return ['atom', '\\\\.', undef, + 'comment', '/\*', '\*/', + 'comment', '//', "\$", + 'string', '"', '"', + 'string', "'", "'", + 'include', '#\s*include', "\$"]; +} + +sub typemap { + return { + 'c' => 'class', + 'd' => 'macro (un)definition', + 'e' => 'enumerator', + 'f' => 'function definition', + 'g' => 'enumeration name', + 'm' => 'class, struct, or union member', + 'n' => 'namespace', + 'p' => 'function prototype or declaration', + 's' => 'structure name', + 't' => 'typedef', + 'u' => 'union name', + 'v' => 'variable definition', + 'x' => 'extern or forward variable declaration', + 'i' => 'interface'}; +} + +sub markuphandlers { + my ($self, $context, $node, $markup) = @_; + + my $index = $context->config->{'index'}; + my $syms = $index->symbols_by_file($context->tree, $context->release, + $node->name); + my $idre = $self->identifier_re(); + + my %subst; + + my $format_newline = $markup->make_format_newline($node); + $subst{'comment'} = new Subst::Complex + qr/\n/ => $format_newline, + qr/[^\n]+/ => sub { $markup->format_comment(@_) }; + + $subst{'string'} = new Subst::Complex + qr/\n/ => $format_newline, + qr/[^\n\"\']+/ => sub { $markup->format_string(@_) }; + + $subst{'include'} = new Subst::Complex + qr/\n/ => $format_newline, + qr/(include\s*\")(.*?)(\")/ => sub { + $markup->format_include([$self->resolve_include($context, $node, @_)], + @_) }, + + qr/(include\s*\<)(.*?)(\>)/ => sub { + $markup->format_include([$self->resolve_include($context, $node, @_)], + @_) }; + + $subst{'code'} = new Subst::Complex + qr/\n/ => $format_newline, + qr/[^\n]*/ => sub { $markup->format_code($idre, $syms, @_) }; + + $subst{'start'} = new Subst::Complex + qr/^/ => $format_newline; + + return \%subst; +} + +sub resolve_include { + my ($self, $context, $node, $frag) = @_; + + if ($frag =~ /include\s+<(.*?)>/) { + return $self->expand_include($context, $node, $1); + } + elsif ($frag =~ /include\s+\"(.*?)\"/) { + my $incl = $1; + my $bare = $1; + my $name = $node->name(); + if ($name =~ /(.*\/)/) { + $incl = $1.$incl; + 1 while $incl =~ s,/[^/]+/../,/,; + + my $file = $context->config->{'repository'}->node($incl, $context->release); + return $incl if $file; + return $self->expand_include($context, $node, $bare); + } + } + + return (); +} + +1; diff --git a/lib/LXRng/Lang/Generic.pm b/lib/LXRng/Lang/Generic.pm new file mode 100644 index 0000000..29443f4 --- /dev/null +++ b/lib/LXRng/Lang/Generic.pm @@ -0,0 +1,22 @@ +package LXRng::Lang::Generic; + +use strict; + +sub expand_include { + my ($self, $context, $node, $include) = @_; + + return () unless $context->config->{'include_maps'}; + + my $file = $node->name(); + foreach my $map (@{$context->config->{'include_maps'}}) { + my @key = $file =~ /($$map[0])/ or next; + my @val = $include =~ /($$map[1])/ or next; + shift(@key); + shift(@val); + my @paths = $$map[2]->(@key, @val); + + return map { /([^\/].*)/ ? $1 : $_ } @paths; + } +} + +1; diff --git a/lib/LXRng/Lang/Undefined.pm b/lib/LXRng/Lang/Undefined.pm new file mode 100644 index 0000000..c4c3c72 --- /dev/null +++ b/lib/LXRng/Lang/Undefined.pm @@ -0,0 +1,45 @@ +package LXRng::Lang::Undefined; + +use strict; +use Subst::Complex; + +use base qw(LXRng::Lang::Generic); + + +sub doindex { + return 0; +} + +sub pathexp { + return qr/$/; +} + +sub reserved { + return {}; +} + +sub parsespec { + return ['atom', '\\\\.', undef]; +} + +sub typemap { + return {}; +} + +sub markuphandlers { + my ($self, $context, $node, $markup) = @_; + + my $format_newline = $markup->make_format_newline($node); + + my %subst; + $subst{'code'} = new Subst::Complex + qr/\n/ => $format_newline, + qr/[^\n]*/ => sub { $markup->format_raw(@_) }; + + $subst{'start'} = new Subst::Complex + qr/^/ => $format_newline; + + return \%subst; +} + +1; diff --git a/lib/LXRng/Markup/Dir.pm b/lib/LXRng/Markup/Dir.pm new file mode 100644 index 0000000..b7743ee --- /dev/null +++ b/lib/LXRng/Markup/Dir.pm @@ -0,0 +1,64 @@ +package LXRng::Markup::Dir; + +use strict; +use POSIX qw(strftime); +use LXRng::Cached; + +sub new { + my ($class, %args) = @_; + + return bless(\%args, $class); +} + +sub context { + my ($self) = @_; + return $$self{'context'}; +} + +sub _format_time { + my ($secs, $zone) = @_; + + my $offset = 0; + if ($zone and $zone =~ /^([-+])(\d\d)(\d\d)$/) { + $offset = ($2 * 60 + $3) * 60; + $offset = -$offset if $1 eq '-'; + $secs += $offset; + } + else { + $zone = ''; + } + return strftime("%F %T $zone", gmtime($secs)); +} + +sub listing { + my ($self) = @_; + + cached { + my @list; + foreach my $n ($$self{'node'}->contents) { + if ($n->isa('LXRng::Repo::Directory')) { + push(@list, {'name' => $n->name, + 'node' => $n->node, + 'size' => '', + 'time' => '', + 'desc' => ''}); + } + else { + my $rfile_id = $self->context->config->{'index'}->rfile_id($n); + my ($s, $tz) = + $self->context->config->{'index'}->get_rfile_timestamp($rfile_id); + ($s, $tz) = $n->time =~ /^(\d+)(?: ([-+]\d\d\d\d)|)$/ + unless $s; + + push(@list, {'name' => $n->name, + 'node' => $n->node, + 'size' => $n->size, + 'time' => _format_time($s, $tz), + 'desc' => ''}); + } + } + \@list; + } $$self{'node'}; +} + +1; diff --git a/lib/LXRng/Markup/File.pm b/lib/LXRng/Markup/File.pm new file mode 100644 index 0000000..406737c --- /dev/null +++ b/lib/LXRng/Markup/File.pm @@ -0,0 +1,120 @@ +package LXRng::Markup::File; + +use strict; +use HTML::Entities; + +sub new { + my ($class, %args) = @_; + + return bless(\%args, $class); +} + +sub context { + my ($self) = @_; + return $$self{'context'}; +} + +sub safe_html { + my ($str) = @_; + return encode_entities($str, '^\n\r\t !\#\$\(-;=?-~'); +} + +sub make_format_newline { + my ($self, $node) = @_; + my $line = 0; + my $tree = $self->context->vtree(); + my $name = $node->name; + + sub { + my ($nl) = @_; + $line++; + $nl = safe_html($nl); + + # id="<num>" is not valid XHTML 1.0, but it is an extremely + # handy shorthand for generating line numbers that don't + # affect cut-n-paste. + return qq{$nl<a id="L$line" name="L$line"></a>}. + qq{<a href="$name#L$line" id="$line" class="line"></a>}; + } +} + +sub format_comment { + my ($self, $com) = @_; + + $com = safe_html($com); + return qq{<span class="comment">$com</span>}; +} + + +sub format_string { + my ($self, $str) = @_; + + $str = safe_html($str); + return qq{<span class="string">$str</span>} +} + +sub format_include { + my ($self, $paths, $all, $pre, $inc, $suf) = @_; + + my $tree = $self->context->vtree(); + if (@$paths > 1) { + $pre = safe_html($pre); + $inc = safe_html($inc); + $suf = safe_html($suf); + my $alts = join("|", map { $_ } @$paths); + return qq{$pre<a href="+ambig=$alts" class="falt">$inc</a>$suf}; + } + elsif (@$paths > 0) { + $pre = safe_html($pre); + $inc = safe_html($inc); + $suf = safe_html($suf); + return qq{$pre<a href="$$paths[0]" class="fref">$inc</a>$suf}; + } + else { + return safe_html($all); + } +} + +sub format_code { + my ($self, $idre, $syms, $frag) = @_; + + my $tree = $self->context->vtree(); + my $path = $self->context->path(); + Subst::Complex::s($frag, + $idre => sub { + my $sym = $_[1]; + if (exists($$syms{$sym})) { + $sym = safe_html($sym); + return qq{<a href="+code=$sym" class="sref">$sym</a>} + } + else { + return safe_html($sym); + } + }, + qr/(.*?)/ => sub { return safe_html($_[0]) }, + ); +} + +sub format_raw { + my ($self, $str) = @_; + + return safe_html($str); +} + +sub markupfile { + my ($self, $subst, $parse) = @_; + + my ($btype, $frag) = $parse->nextfrag; + + return () unless defined $frag; + + $btype ||= 'code'; + if ($btype and exists $$subst{$btype}) { + return $$subst{$btype}->s($frag); + } + else { + return $frag; + } +} + +1; diff --git a/lib/LXRng/Parse/Simple.pm b/lib/LXRng/Parse/Simple.pm new file mode 100644 index 0000000..53f7e10 --- /dev/null +++ b/lib/LXRng/Parse/Simple.pm @@ -0,0 +1,127 @@ +package LXRng::Parse::Simple; + +use strict; +use integer; +use IO::Handle; + +sub new { + my ($class, $fileh, $tabhint, @blksep) = @_; + + my (@bodyid, @open, @term); + + while (my @a = splice(@blksep,0,3)) { + push(@bodyid, $a[0]); + push(@open, $a[1]); + push(@term, $a[2]); + } + + my $self = { + 'fileh' => $fileh, # File handle + 'tabwidth' => $tabhint||8, # Tab width + 'frags' => [], # Fragments in queue + 'bodyid' => \@bodyid, # Array of body type ids + 'bofseen' => 0, # Beginning-of-file seen? + 'term' => \@term, + # Fragment closing delimiters + 'open' => join('|', map { "($_)" } @open), + # Fragment opening regexp + 'split' => join('|', @open, map { $_ eq '' ? () : $_ } @term), + # Fragmentation regexp + }; + + return bless $self, $class; + +# @frags $fileh $tabwidth $split @term $open $bodyid + +} + +sub untabify { + my $t = $_[1] || 8; + + $_[0] =~ s/^(\t+)/(' ' x ($t * length($1)))/ge; # Optimize for common case. + $_[0] =~ s/([^\t]*)\t/$1.(' ' x ($t - (length($1) % $t)))/ge; + return($_[0]); +} + + +sub nextfrag { + my ($self) = @_; + + my $btype = undef; + my $frag = undef; + my $line = ''; + + while (1) { + # read one more line if we have processed + # all of the previously read line + if (@{$$self{'frags'}} == 0) { + $line = $$self{'fileh'}->getline; + + if ($. <= 2 && + $line =~ /^.*-[*]-.*?[ \t;]tab-width:[ \t]*([0-9]+).*-[*]-/) { + # make sure there really is a non-zero tabwidth + $$self{'tabwidth'} = $1 if $1 > 0; + } + + if(defined($line)) { + untabify($line, $$self{'tabwidth'}); + + # split the line into fragments + $$self{'frags'} = [split(/($$self{'split'})/, $line)]; + } + } + + last if @{$$self{'frags'}} == 0; + + unless ($$self{'bofseen'}) { + # return start marker if file has contents + $$self{'bofseen'} = 1; + return ('start', ''); + } + + # skip empty fragments + if ($$self{'frags'}[0] eq '') { + shift(@{$$self{'frags'}}); + } + + # check if we are inside a fragment + if (defined($frag)) { + if (defined($btype)) { + my $next = shift(@{$$self{'frags'}}); + + # Add to the fragment + $frag .= $next; + # We are done if this was the terminator + last if $next =~ /^$$self{'term'}[$btype]$/; + + } + else { + if ($$self{'frags'}[0] =~ /^$$self{'open'}$/) { + last; + } + $frag .= shift(@{$$self{'frags'}}); + } + } + else { + # Find the blocktype of the current block + $frag = shift(@{$$self{'frags'}}); + if (defined($frag) && (@_ = $frag =~ /^$$self{'open'}$/)) { + # grep in a scalar context returns the number of times + # EXPR evaluates to true, which is this case will be + # the index of the first defined element in @_. + + my $i = 1; + $btype = grep { $i &&= !defined($_) } @_; + if(!defined($$self{'term'}[$btype])) { + # Opening regexp captures entire block. + last; + } + } + } + } + $btype = $$self{'bodyid'}[$btype] if defined($btype); + + return ($btype, $frag); +} + +1; diff --git a/lib/LXRng/Repo/Directory.pm b/lib/LXRng/Repo/Directory.pm new file mode 100644 index 0000000..ad892b2 --- /dev/null +++ b/lib/LXRng/Repo/Directory.pm @@ -0,0 +1,17 @@ +package LXRng::Repo::Directory; + +use strict; + +sub name { + my ($self) = @_; + + return $$self{'name'}; +} + +sub node { + my ($self) = @_; + + $self->name =~ m,([^/]+)/?$, and return $1; +} + +1; diff --git a/lib/LXRng/Repo/File.pm b/lib/LXRng/Repo/File.pm new file mode 100644 index 0000000..7df5a04 --- /dev/null +++ b/lib/LXRng/Repo/File.pm @@ -0,0 +1,17 @@ +package LXRng::Repo::File; + +use strict; + +sub name { + my ($self) = @_; + + return $$self{'name'}; +} + +sub node { + my ($self) = @_; + + $self->name =~ m,([^/]+)$, and return $1; +} + +1; diff --git a/lib/LXRng/Repo/Git.pm b/lib/LXRng/Repo/Git.pm new file mode 100644 index 0000000..2d6ea33 --- /dev/null +++ b/lib/LXRng/Repo/Git.pm @@ -0,0 +1,111 @@ +package LXRng::Repo::Git; + +use strict; +use Memoize; +use LXRng::Cached; +use LXRng::Repo::Git::Iterator; +use LXRng::Repo::Git::File; +use LXRng::Repo::Git::Directory; + +sub _git_cmd { + my ($self, $cmd, @args) = @_; + + my $git; + my $pid = open($git, "-|"); + die $! unless defined $pid; + if ($pid == 0) { + $ENV{'GIT_DIR'} = $$self{'root'}; + exec('git', $cmd, @args); + warn $!; + kill(9, $$); + } + return $git; +} + +sub new { + my ($class, $root, %args) = @_; + + memoize('_release_timestamp'); + + return bless({root => $root, %args}, $class); +} + +sub _release_timestamp { + my ($self, $release) = @_; + + my $cinfo = $self->_git_cmd('cat-file', 'commit', $release); + + my $time; + while (<$cinfo>) { + $time = $1 if /^author .*? (\d+(?: [-+]\d+|))$/ ; + $time ||= $1 if /^committer .*? (\d+(?: [-+]\d+|))$/ ; + } + + return $time; +} + +sub _use_author_timestamp { + my ($self) = @_; + + return $$self{'author_timestamp'}; +} + +sub _sort_key { + my ($v) = @_; + + $v =~ s/(\d+)/sprintf("%05d", $1)/ge; + return $v; +} + +sub allversions { + my ($self) = @_; + + cached { + my @tags; + my $tags = $self->_git_cmd('tag', '-l'); + while (<$tags>) { + chomp; + next if $$self{'release_re'} and $_ !~ $$self{'release_re'}; + push(@tags, $_); + } + + return (sort {_sort_key($b) cmp _sort_key($a) } @tags); + }; +} + +sub node { + my ($self, $path, $release) = @_; + + $path =~ s,^/+,,; + $path =~ s,/+$,,; + + if ($path eq '') { + open(my $tag, '<', $$self{'root'}.'/refs/tags/'.$release) + or return undef; + my $ref = <$tag>; + close($tag); + chomp($ref); + return LXRng::Repo::Git::Directory->new($self, '', $ref); + } + + my $git = $self->_git_cmd('ls-tree', $release, $path); + my ($mode, $type, $ref, $gitpath) = split(" ", <$git>); + + if ($type eq 'tree') { + return LXRng::Repo::Git::Directory->new($self, $path, $ref, $release); + } + elsif ($type eq 'blob') { + return LXRng::Repo::Git::File->new($self, $path, $ref, $release); + } + else { + return undef; + } +} + +sub iterator { + my ($self, $release) = @_; + + return LXRng::Repo::Git::Iterator->new($self, $release); +} + +1; diff --git a/lib/LXRng/Repo/Git/Directory.pm b/lib/LXRng/Repo/Git/Directory.pm new file mode 100644 index 0000000..592e608 --- /dev/null +++ b/lib/LXRng/Repo/Git/Directory.pm @@ -0,0 +1,56 @@ +package LXRng::Repo::Git::Directory; + +use strict; + +use base qw(LXRng::Repo::Directory); + +sub new { + my ($class, $repo, $name, $ref, $rel) = @_; + + $name =~ s,/*$,/,; + return bless({repo => $repo, name => $name, ref => $ref, rel => $rel}, + $class); +} + +sub time { + my ($self) = @_; + + return 0; +# return $$self{'stat'}[9]; +} + +sub size { + my ($self) = @_; + + return ''; +} + +sub contents { + my ($self) = @_; + + my $git = $$self{'repo'}->_git_cmd('ls-tree', $$self{'ref'}); + + my $prefix = $$self{'name'}; + $prefix =~ s,^/+,,; + my (@dirs, @files); + while (<$git>) { + chomp; + my ($mode, $type, $ref, $node) = split(" ", $_); + if ($type eq 'tree') { + push(@dirs, LXRng::Repo::Git::Directory->new($$self{'repo'}, + $prefix.$node, + $ref, + $$self{'rel'})); + } + elsif ($type eq 'blob') { + push(@files, LXRng::Repo::Git::File->new($$self{'repo'}, + $prefix.$node, + $ref, + $$self{'rel'})); + } + } + + return (@dirs, @files); +} + +1; diff --git a/lib/LXRng/Repo/Git/File.pm b/lib/LXRng/Repo/Git/File.pm new file mode 100644 index 0000000..b0bb9a3 --- /dev/null +++ b/lib/LXRng/Repo/Git/File.pm @@ -0,0 +1,80 @@ +package LXRng::Repo::Git::File; + +use strict; + +use base qw(LXRng::Repo::File); +use LXRng::Repo::TmpFile; +use File::Temp qw(tempdir); + +sub new { + my ($class, $repo, $name, $ref, $rel) = @_; + + return bless({repo => $repo, name => $name, ref => $ref, rel => $rel}, + $class); +} + +sub time { + my ($self) = @_; + + if ($$self{'repo'}->_use_author_timestamp) { + # This is painfully slow. It is only performed index-time, + # but that might stil be bad enough that you would want to + # just use the release-timestamp insted. + my $cinfo = $$self{'repo'}->_git_cmd('log', '--pretty=raw', + '--max-count=1', '--all', + '..'.$$self{'rel'}, + '--', $self->name); + + my $time; + while (<$cinfo>) { + $time = $1 if /^author .*? (\d+(?: [-+]\d+|))$/ ; + $time ||= $1 if /^committer .*? (\d+(?: [-+]\d+|))$/ ; + } + + return $time if $time; + } + + return $$self{'repo'}->_release_timestamp($$self{'rel'}); +} + +sub size { + my ($self) = @_; + + my $git = $$self{'repo'}->_git_cmd('cat-file', '-s', $$self{'ref'}); + my $size = <$git>; + close($git); + chomp($size); + return $size; +} + +sub handle { + my ($self) = @_; + + return $$self{'repo'}->_git_cmd('cat-file', 'blob', $$self{'ref'}); +} + +sub revision { + my ($self) = @_; + + return $$self{'ref'}; +} + +sub phys_path { + my ($self) = @_; + + my $tmpdir = tempdir() or die($!); + open(my $phys, ">", $tmpdir.'/'.$self->node) or die($!); + + my $handle = $self->handle(); + my $buf = ''; + while (sysread($handle, $buf, 64*1024) > 0) { + print($phys $buf) or die($!); + } + close($handle); + close($phys) or die($!); + + return LXRng::Repo::TmpFile->new(dir => $tmpdir, + node => $self->node); +} + +1; diff --git a/lib/LXRng/Repo/Git/Iterator.pm b/lib/LXRng/Repo/Git/Iterator.pm new file mode 100644 index 0000000..978e584 --- /dev/null +++ b/lib/LXRng/Repo/Git/Iterator.pm @@ -0,0 +1,33 @@ +package LXRng::Repo::Git::Iterator; + +use strict; +use LXRng::Repo::Git::File; + +sub new { + my ($class, $repo, $release) = @_; + + my @refs; + my $git = $repo->_git_cmd('ls-tree', '-r', $release); + while (<$git>) { + if (/\S+\s+blob\s+(\S+)\s+(\S+)/) { + push(@refs, [$2, $1]); + } + } + close($git); + + return bless({refs => \@refs, repo => $repo, rel => $release}, $class); +} + +sub next { + my ($self) = @_; + + return undef unless @{$$self{'refs'}} > 0; + my $file = shift(@{$$self{'refs'}}); + + return LXRng::Repo::Git::File->new($$self{'repo'}, + $$file[0], + $$file[1], + $$self{'rel'}); +} + +1; diff --git a/lib/LXRng/Repo/Git/TarFile.pm b/lib/LXRng/Repo/Git/TarFile.pm new file mode 100644 index 0000000..33af87d --- /dev/null +++ b/lib/LXRng/Repo/Git/TarFile.pm @@ -0,0 +1,98 @@ +package LXRng::Repo::Git::TarFile; + +use strict; +use File::Temp qw(tempdir); +use Fcntl qw(F_GETFD F_SETFD FD_CLOEXEC); + +use base qw(LXRng::Repo::File); + +sub new { + my ($class, $tar, $ref) = @_; + + return bless({tar => $tar, ref => $ref}, $class); +} + +sub name { + my ($self) = @_; + + return $$self{'tar'}->name(); +} + +sub node { + my ($self) = @_; + + $self->name =~ m,.*/([^/]+), and return $1; +} + +sub time { + my ($self) = @_; + + return $$self{'tar'}->mtime(); +} + +sub size { + my ($self) = @_; + + return $$self{'tar'}->size; +} + +sub phys_path { + my ($self) = @_; + + my $tmpdir = tempdir() or die($!); + open(my $phys, ">", $tmpdir.'/'.$self->node); + + my $data = $$self{'tar'}->get_content_by_ref(); + my $len = $$self{'tar'}->size(); + my $pos = 0; + while ($pos < $len) { + print($phys substr($$data, $pos, 64*1024)); + $pos += 64*1024; + } + close($phys); + + return LXRng::Repo::Git::TarFile::Virtual->new(dir => $tmpdir, + node => $self->node); +} + +sub handle { + my ($self) = @_; + + my $data = $$self{'tar'}->get_content_by_ref(); + open(my $fh, "<", $data); + + return $fh; +} + +sub revision { + my ($self) = @_; + + $$self{'ref'} ||= $self->time.'.'.$self->size; + return $$self{'ref'}; +} + +package LXRng::Repo::Git::TarFile::Virtual; + +use strict; +use overload '""' => \&filename; + +sub new { + my ($class, %args) = @_; + + return bless(\%args, $class); +} + +sub filename { + my ($self) = @_; + + return $$self{'dir'}.'/'.$$self{'node'}; +} + +sub DESTROY { + my ($self) = @_; + unlink($$self{'dir'}.'/'.$$self{'node'}); + rmdir($$self{'dir'}); +# kill(9, $$self{'pid'}); +} + +1; diff --git a/lib/LXRng/Repo/Plain.pm b/lib/LXRng/Repo/Plain.pm new file mode 100644 index 0000000..c30835e --- /dev/null +++ b/lib/LXRng/Repo/Plain.pm @@ -0,0 +1,38 @@ +package LXRng::Repo::Plain; + +use strict; +use LXRng::Repo::Plain::Iterator; +use LXRng::Repo::Plain::File; +use LXRng::Repo::Plain::Directory; + +sub new { + my ($class, $root) = @_; + + return bless({root => $root}, $class); +} + +sub allversions { + my ($self) = @_; + + my @ver = (sort + grep { $_ ne "." and $_ ne ".." } + map { substr($_, length($$self{'root'})) =~ /([^\/]*)/; $1 } + glob($$self{'root'}."*/")); + + return @ver; +} + +sub node { + my ($self, $path, $release) = @_; + + my $realpath = join('/', $$self{'root'}, $release, $path); + return LXRng::Repo::Plain::File->new($path, $realpath); +} + +sub iterator { + my ($self, $release) = @_; + + return LXRng::Repo::Plain::Iterator->new($self->node('', $release)); +} + +1; diff --git a/lib/LXRng/Repo/Plain/Directory.pm b/lib/LXRng/Repo/Plain/Directory.pm new file mode 100644 index 0000000..8d7e701 --- /dev/null +++ b/lib/LXRng/Repo/Plain/Directory.pm @@ -0,0 +1,45 @@ +package LXRng::Repo::Plain::Directory; + +use strict; + +use base qw(LXRng::Repo::Directory); + +sub new { + my ($class, $name, $path, $stat) = @_; + + $name =~ s,(.)/*$,$1/,; + $path =~ s,/*$,/,; + return bless({name => $name, path => $path, stat => $stat}, $class); +} + +sub time { + my ($self) = @_; + + return $$self{'stat'}[9]; +} + +sub size { + my ($self) = @_; + + return ''; +} + +sub contents { + my ($self) = @_; + + my (@dirs, @files); + my ($dir, $node); + opendir($dir, $$self{'path'}) or die("Can't open ".$$self{'path'}.": $!"); + while (defined($node = readdir($dir))) { + next if $node =~ /^\.|~$|\.orig$/; + next if $node eq 'CVS'; + + push(@files, LXRng::Repo::Plain::File->new($$self{'name'}.$node, + $$self{'path'}.$node)); + } + closedir($dir); + + return sort { ref($a) cmp ref($b) || $$a{'name'} cmp $$b{'name'} } @files; +} + +1; diff --git a/lib/LXRng/Repo/Plain/File.pm b/lib/LXRng/Repo/Plain/File.pm new file mode 100644 index 0000000..cf2d6d5 --- /dev/null +++ b/lib/LXRng/Repo/Plain/File.pm @@ -0,0 +1,51 @@ +package LXRng::Repo::Plain::File; + +use strict; + +use base qw(LXRng::Repo::File); +use Fcntl; + +sub new { + my ($class, $name, $path) = @_; + + my @stat = stat($path); + + return undef unless @stat; + + return LXRng::Repo::Plain::Directory->new($name, $path, \@stat) if -d _; + + return bless({name => $name, path => $path, stat => \@stat}, $class); +} + +sub time { + my ($self) = @_; + + return $$self{'stat'}[9]; +} + +sub size { + my ($self) = @_; + + return $$self{'stat'}[7]; +} + +sub phys_path { + my ($self) = @_; + + return $$self{'path'}; +} + +sub revision { + my ($self) = @_; + + return $self->time.'.'.$self->size; +} + +sub handle { + my ($self) = @_; + + sysopen(my $handle, $self->phys_path, O_RDONLY) or die($!); + return $handle; +} + +1; diff --git a/lib/LXRng/Repo/Plain/Iterator.pm b/lib/LXRng/Repo/Plain/Iterator.pm new file mode 100644 index 0000000..b086860 --- /dev/null +++ b/lib/LXRng/Repo/Plain/Iterator.pm @@ -0,0 +1,29 @@ +package LXRng::Repo::Plain::Iterator; + +use strict; +use LXRng::Repo::Plain; + +sub new { + my ($class, $dir) = @_; + + return bless({dir => $dir, stack => [], nodes => [$dir->contents]}, $class); +} + +sub next { + my ($self) = @_; + + while (@{$$self{'nodes'}} == 0) { + return undef unless @{$$self{'stack'}}; + $$self{'nodes'} = pop(@{$$self{'stack'}}); + } + + my $node = shift(@{$$self{'nodes'}}); + if ($node->isa('LXRng::Repo::Directory')) { + push(@{$$self{'stack'}}, $$self{'nodes'}); + $$self{'nodes'} = [$node->contents]; + return $self->next; + } + return $node; +} + +1; diff --git a/lib/LXRng/Repo/TmpFile.pm b/lib/LXRng/Repo/TmpFile.pm new file mode 100644 index 0000000..bc9024a --- /dev/null +++ b/lib/LXRng/Repo/TmpFile.pm @@ -0,0 +1,30 @@ +package LXRng::Repo::TmpFile; + +# This package is used to hold on to a reference to a physical copy of +# a file normally only present inside a repo of some sort. When it +# leaves scopy, the destructor will remove it. (The object acts as +# string containing the path of the physical manifestation of the +# file.) + +use strict; +use overload '""' => \&filename; + +sub new { + my ($class, %args) = @_; + + return bless(\%args, $class); +} + +sub filename { + my ($self) = @_; + + return $$self{'dir'}.'/'.$$self{'node'}; +} + +sub DESTROY { + my ($self) = @_; + unlink($$self{'dir'}.'/'.$$self{'node'}); + rmdir($$self{'dir'}); +} + +1; diff --git a/lib/LXRng/Search/Xapian.pm b/lib/LXRng/Search/Xapian.pm new file mode 100644 index 0000000..42c7580 --- /dev/null +++ b/lib/LXRng/Search/Xapian.pm @@ -0,0 +1,152 @@ +package LXRng::Search::Xapian; + +use strict; +use Search::Xapian qw/:ops :db :qpstem/; +use Search::Xapian::QueryParser; + + +sub new { + my ($class, $db_root) = @_; + + $ENV{'XAPIAN_PREFER_FLINT'} = 1; + my $self = bless({'db_root' => $db_root, + 'writes' => 0}, + $class); + + return $self; +} + +sub wrdb { + my ($self) = @_; + + return $$self{'wrdb'} ||= Search::Xapian::WritableDatabase + ->new($$self{'db_root'}, Search::Xapian::DB_CREATE_OR_OPEN); +} + +sub new_document { + my ($self, $desc) = @_; + + my $doc = Search::Xapian::Document->new(); + $doc->set_data($desc); + return $doc; +} + +sub add_document { + my ($self, $doc, $rel_id) = @_; + + $doc->add_term('__@@LXRREL_'.$rel_id); + my $doc_id = $self->wrdb->add_document($doc); + $self->{'writes'}++; + $self->flush() if $self->{'writes'} % 499 == 0; + return $doc_id; +} + +sub add_release { + my ($self, $doc_id, $rel_id) = @_; + + my $reltag = '__@@LXRREL_'.$rel_id; + my $doc = $self->wrdb->get_document($doc_id); + + my $term = $doc->termlist_begin; + my $termend = $doc->termlist_end; + $term->skip_to($reltag); + if ($term ne $termend) { + return 0 if $term->get_termname eq $reltag; + } + $doc->add_term($reltag); + $self->wrdb->replace_document($doc_id, $doc); + return 1; +} + +sub flush { + my ($self) = @_; + + warn "\n*** hash: flushing\n"; + $self->wrdb->flush(); +} + +sub search { + my ($self, $rel_id, $query) = @_; + + my $db = Search::Xapian::Database->new($$self{'db_root'}); + my $qp = new Search::Xapian::QueryParser($db); + $qp->set_stemming_strategy(STEM_NONE); + $qp->set_default_op(OP_AND); + + if ($query =~ /\"/) { + # Only moderate fixup of advanced queries + $query =~ s/\b([A-Z]+)\b/\L$1\E/g; + } + else { + $query =~ s/([\S_]+_[\S_]*)/\"$1\"/g; + $query =~ s/_/ /g; + $query =~ s/\b(?![A-Z][^A-Z]*\b)(\S+)/\L$1\E/g; + } + + my $query = $qp->parse_query($query); + $query = Search::Xapian::Query + ->new(OP_FILTER, $query, + Search::Xapian::Query->new('__@@LXRREL_'.$rel_id)); + + my $enq = $db->enquire($query); + + my $matches = $enq->get_mset(0, 100); + my $total = $matches->get_matches_estimated(); + my $size = $matches->size(); + + my @res; + + my $match = $matches->begin(); + my $i = 0; + while ($i++ < $size) { + my $term = $enq->get_matching_terms_begin($match); + my $termend = $enq->get_matching_terms_end($match); + my %lines; + my $hits = 0; + while ($term ne $termend) { + if ($term !~ /^__\@\@LXR/) { + my $pos = $db->positionlist_begin($match->get_docid(), $term); + my $posend = $db->positionlist_end($match->get_docid(), $term); + while ($pos ne $posend) { + $lines{int($pos/100)}{$term} = 1; + $hits++; + $pos++; + } + } + $term++; + } + # Sort lines in order of the most matching terms + my %byhits; + foreach my $l (keys %lines) { + $byhits{0+keys(%{$lines{$l}})}{$l} = 1; + } + # Only consider the lines having the max number of terms + my ($max) = sort { $b <=> $a } keys %byhits; + my @lines = sort keys(%{$byhits{$max}}); + + push(@res, [$match->get_percent(), + $match->get_document->get_data(), + $lines[0], + 0+@lines]) + if @lines; + $match++; + } + + return ($total, \@res); +} + +sub reset_db { + my ($self) = @_; + + foreach my $f (glob($$self{'db_root'}.'/*')) { + unlink($f); + } +} + +sub DESTROY { + my ($self) = @_; + + $self->flush() if $self->{'writes'} > 0; +} + +1; diff --git a/lib/Subst/Complex.pm b/lib/Subst/Complex.pm new file mode 100644 index 0000000..788d469 --- /dev/null +++ b/lib/Subst/Complex.pm @@ -0,0 +1,63 @@ +package Subst::Complex; + +use strict; + +sub new { + my ($self, @args) = @_; + + my (@re, @ac); + my $l = 1; + while (@args) { + my ($r, $a) = splice(@args, 0, 2); + "" =~ /|$r/; + + push(@ac, [$l+1, $l+1+$#+, $a]); + $l += 1+$#+; + + push(@re, "($r)"); + } + + return bless {'re' => '((?s:.*?))(?:'.join('|', @re).')', + 'ac' => \@ac}, $self; +} + +sub s { + my $self; + my $str; + + if (ref($_[0])) { + $self = shift; + $str = shift; + } + else { + $str = shift; + $self = __PACKAGE__->new(@_); + } + + my @res; +# $str =~ s{$$self{'re'}}{ +# push(@res, $1) if length($1); +# my ($a) = grep { defined $-[$$_[0]] } @{$$self{'ac'}}; +# my @g = map { substr($str, $-[$_], $+[$_] - $-[$_]); +# } $$a[0]..$$a[1]; +# push(@res, $$a[2]->(@g)); +# ''; +# }ge; + $str =~ s{$$self{'re'}}{ + push(@res, $1) if length($1); + my ($a) = grep { defined $-[$$_[0]] } @{$$self{'ac'}}; + my @g = map { substr($str, $-[$_], $+[$_] - $-[$_]); + } $$a[0]..$$a[1]; + push(@res, [$$a[2], [@g]]); + ''; + }ge; + + push(@res, $str) if length($str); + @res = map { + ref($_) ? $$_[0]->(@{$$_[1]}) : $_ + } @res; + + return @res; +} + +1; diff --git a/lxr-db-admin b/lxr-db-admin new file mode 100755 index 0000000..7d6023c --- /dev/null +++ b/lxr-db-admin @@ -0,0 +1,40 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::Bin/lib"; + +use Getopt::Long; + +use LXRng ROOT => $FindBin::Bin; +use LXRng::Context; + +my $reset = 0; +my $drop = 0; +my $init = 0; + +GetOptions("reset" => \$reset, + "drop" => \$drop, + "init" => \$init) + or die "Failed to parse options"; + +my $tree = $ARGV[0]; +my $context = LXRng::Context->new('tree' => $tree) + if $tree; + +die "Usage: $0 <tree-id> [--init | --reset | --drop]\n" + unless $context and ($reset or $drop or $init); + +my $index = $context->config->{'index'}; +my $hash = $context->config->{'search'}; + +if ($reset or $drop) { + $index->drop_db(); + $hash->reset_db(); +} +if ($reset or $init) { + $index->init_db(); +} + diff --git a/lxr-genxref b/lxr-genxref new file mode 100755 index 0000000..79e98f5 --- /dev/null +++ b/lxr-genxref @@ -0,0 +1,301 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::Bin/lib"; + +use LXRng ROOT => $FindBin::Bin; +use LXRng::Context; +use LXRng::Lang qw(C); +use LXRng::Index; +use LXRng::Parse::Simple; + +use Carp; +use Data::Dumper; +use IO::Handle; +use Fcntl; + +$SIG{'INT'} = sub { die "SIGINT: please wait, flushing caches...\n"; }; +$SIG{'QUIT'} = sub { die "SIGQUIT: please wait, flushing caches...\n"; }; +$SIG{'TERM'} = sub { die "SIGTERM: please wait, flushing caches...\n"; }; + +autoflush STDOUT 1; +autoflush STDERR 1; + +my $cols = 0; +sub progress_mark { + my ($mark) = @_; + + if ($cols > 79) { + print("\n"); + $cols = 0; + } + print(STDERR $mark); + $cols++; +} + +sub progress_info { + my ($msg) = @_; + print(STDERR "\n") if $cols > 0; + print(STDERR "$msg\n"); + $cols = 0; +} + +sub make_add_ident { + my ($index, $fileid) = @_; + + my $last_func; + my %identcache; + + my $add_ident; + $add_ident = sub { + my ($symbol, $info) = @_; + + if ($$info{'kind'} eq 'f') { + $last_func = $symbol; + } + if ($$info{'kind'} eq 'l') { + $$info{'context'} = $identcache{$last_func}; + } + if (exists $$info{'class'}) { + $$info{'context'} = $identcache{$$info{'class'}}; + } + if (exists $$info{'struct'}) { + $$info{'context'} = $identcache{$$info{'struct'}}; + } + + $identcache{$symbol} = + $index->add_ident($fileid, $$info{'line'}, + $symbol, $$info{'kind'}, + $$info{'context'}); + + if ($$info{'kind'} eq 'f' and exists $$info{'signature'}) { + # This needs to be more robust. Perhaps ctags ought to do it. + + foreach my $v (split(/,/, $$info{'signature'})) { + next if + $v !~ /([a-zA-Z_0-9]+)[^a-zA-Z_0-9]*$/ or + $1 eq 'void'; + + $add_ident->($1, {'kind' => 'l', 'line' => $$info{'line'}}); + } + } + } +} + +sub index_file { + my ($context, $index, $tree, $rel, $file, $fileid) = @_; + + my $lang = LXRng::Lang->new($file); + + return unless $lang->doindex(); + unless ($index->to_index($fileid)) { + progress_mark("*"); + return; + } + + my $add_ident = make_add_ident($index, $fileid); + + progress_info("indexing ".$file->name."[".$file->revision."] ". + $file->size." bytes ($lang)..."); + + my @extra_flags = ('-IEXPORT_SYMBOL+', '-I__initcall+'); + + my $path = $file->phys_path; + + my $ctags; + my $pid = open($ctags, '-|'); + die $! unless defined $pid; + + if ($pid == 0) { + exec('ctags-exuberant', + @extra_flags, + '--fields=+aifmknsSz', '--sort=no', + '--excmd=number', '-f', '-', + '--language-force='.$lang->ctagslangname, + $lang->ctagsopts, + $path); + + # Still here? + warn $!; + kill(9, $$); + } + + LXRng::Index::transaction { + while (<$ctags>) { + chomp; + my ($symbol, $file, $excmd, @info) = split(/\t/); + my %info = map { split(/:/, $_, 2) } @info; + + $add_ident->($symbol, \%info); + } + } $index; + + $path = undef; +} + +sub reference_file { + my ($context, $index, $tree, $rel, $file, $fileid) = @_; + + my $lang = LXRng::Lang->new($file); + + return unless $lang->doindex(); + + unless ($index->to_reference($fileid)) { + progress_mark("."); + return; + } + +# sysopen(my $handle, $file->phys_path, O_RDONLY) or die($!); +# my $parse = new LXRng::Parse::Simple($handle, 8, @{$lang->parsespec}); + my $parse = new LXRng::Parse::Simple($file->handle, 8, @{$lang->parsespec}); + + progress_info("referencing ".$file->name.", ". + $file->size." bytes ($lang)..."); + + my $res = $lang->reserved(); + + my $re = qr( + (?m:^|[^a-zA-Z0-9_]) # Non-symbol chars. + (_*[a-zA-Z][a-zA-Z0-9_]*) # The symbol. + \b + )x; + + my $line = 1; + while (1) { + my ($btype, $frag) = $parse->nextfrag; + + last unless defined $frag; + $btype ||= 'code'; + if ($btype eq 'code') { + + while ($frag =~ /\G.*?(?:(\n)|$re)/gc) { + $line++ && next if defined $1; + + my $id = $2; + + next if $$res{$id}; + + $index->add_usage($fileid, $line, $id); + } + } + else { + if ($btype eq 'include') { + my @paths = + $lang->resolve_include($context, $file, $frag); + + foreach my $path (@paths) { + $index->add_include($fileid, $path); + } + } + $line += $frag =~ tr/\n/\n/; + } + } +} + +sub hash_file { + my ($context, $index, $hash, $tree, $rel, $file, $fileid) = @_; + + my $docid; + if ($index->to_hash($fileid)) { + my $handle = $file->handle(); + progress_info("hashing ".$file->name."[".$file->revision."] ". + $file->size." bytes..."); + my $doc = $hash->new_document($file->name); + while (<$handle>) { + my $pos = 0; + # Latin-1 word characters. + foreach my $term (/([0-9a-zA-Z\300-\326\330-\366\370-\377]+)/g) { + if ($term =~ /^[A-Z][^A-Z]*$/) { + $term = 'R'.lc($term); + } + else { + $term = lc($term); + } + next if length($term) > 128; + $doc->add_posting($term, $.*100 + $pos++); + } + } + $docid = $hash->add_document($doc, $index->release_id($tree, $rel)); + $index->add_hashed_document($fileid, $docid); + } + else { + $docid = $index->get_hashed_document($fileid); + if ($hash->add_release($docid, $index->release_id($tree, $rel))) { + progress_mark("+"); + } + else { + progress_mark("-"); + } + } + # for all releases this fileid belongs to (that are not is_indexed) + # add_release to $docid. +} + +sub do_index { + my ($context, $index, $hash, $tree, $rel, $iter) = @_; + + my $node; + while (defined($node = $iter->next)) { + next if $node->name =~ /\.o$/; + my $fileid = $index->rfile_id($node, 1); + $index->add_filerelease($tree, $rel, $fileid); + index_file($context, $index, $tree, $rel, $node, $fileid); + hash_file($context, $index, $hash, $tree, $rel, $node, $fileid); + } +} + +sub do_reference { + my ($context, $index, $hash, $tree, $rel, $iter) = @_; + + my $node; + while (defined($node = $iter->next)) { + next if $node->name =~ /\.o$/; + my $fileid = $index->rfile_id($node, 1); + LXRng::Index::transaction { + reference_file($context, $index, $tree, $rel, $node, $fileid); + } $index; + } +} + +my $tree = shift(@ARGV); +my @versions = @ARGV; + +my $context = LXRng::Context->new('tree' => $tree); + +my $index = $context->config->{'index'}; +my $hash = $context->config->{'search'}; +my $rep = $context->config->{'repository'}; + +sub do_genxref { + my ($tree, $version) = @_; + + print("\nindexing release $version...\n"); + my $root = $rep->node('/', $version) or die "bad root"; + $context->release($version); + + LXRng::Index::transaction { + do_index($context, $index, $hash, $tree, $version, $rep->iterator($version)); + do_reference($context, $index, $hash, $tree, $version, $rep->iterator($version)); + } $index; + + $hash->flush(); + # Mark release as is_indexed + progress_info("release $version done"); +} + +if (@versions) { + foreach my $version (@versions) { + do_genxref($tree, $version); + } +} +else { + # Run pass over all un-indexed releases, record all files. + + foreach my $version (reverse @{$context->all_releases}) { + next if $index->_get_release($index->tree_id($tree), $version); + system($0, $tree, $version); + } +} diff --git a/lxrng.conf-dist b/lxrng.conf-dist new file mode 100644 index 0000000..80c35f5 --- /dev/null +++ b/lxrng.conf-dist @@ -0,0 +1,53 @@ +# -*- mode: perl -*- +# Configuration file +# +# + +use LXRng::Index::PgBatch; +use LXRng::Repo::Git; +use LXRng::Search::Xapian; + +my $gitrepo = LXRng::Repo::Git + ->new('/var/lib/lxrng/repos/linux-2.6/.git', + release_re => qr/^v[^-]*$/, + author_timestamp => 0); + +my $index = LXRng::Index::PgBatch->new(db_spec => 'dbname=lxrng;port=5432', + db_user => "", db_pass => "", + # table_prefix => 'lxr' + ); +my $search = LXRng::Search::Xapian->new('/var/lib/lxrng/text-db/linux-2.6'); + +return { + 'linux' => { + 'repository' => $gitrepo, + 'index' => $index, + 'search' => $search, + + 'base_url' => 'http://lxr-test.linpro.no/', + # Must be writable by httpd user: + 'cache' => '/var/lib/lxrng/cache', + + 'fs_charset' => 'iso-8859-1', + 'content_charset' => 'iso-8859-1', + + 'languages' => ['C'], + 'ver_list' => [$gitrepo->allversions], + + 'ver_default' => 'v2.6.20.3', + + 'include_maps' => + [ + [qr|^arch/(.*?)/|, qr|^asm/(.*)|, + sub { "include/asm-$_[0]/$_[1]" }], + [qr|^include/asm-(.*?)/|, qr|^asm/(.*)|, + sub { "include/asm-$_[0]/$_[1]" }], + [qr|^|, qr|^asm/(.*)|, + sub { map { "include/asm-$_/$_[0]" } + qw(i386 alpha arm ia64 m68k mips mips64), + qw(ppc s390 sh sparc sparc64 x86_64) }], + [qr|^|, qr|(.*)|, + sub { "include/$_[0]" }], + ], + }, +}; diff --git a/tmpl/content_dir.tt2 b/tmpl/content_dir.tt2 new file mode 100644 index 0000000..f5bab3f --- /dev/null +++ b/tmpl/content_dir.tt2 @@ -0,0 +1,11 @@ +<table class="directory"> +[% FOREACH link = dir_listing %] +<tr class="node"> +<td class="name"> + <a href="[% link.name %]" class="fref">[% link.node %]</a></td> +<td class="size">[% link.size %]</td> +<td class="time">[% link.time %]</td> +<td class="desc">[% link.desc %]</td> +</tr> +[% END %] +</table> diff --git a/tmpl/footer.tt2 b/tmpl/footer.tt2 new file mode 100644 index 0000000..11a09ca --- /dev/null +++ b/tmpl/footer.tt2 @@ -0,0 +1,3 @@ + +</body> +</html> diff --git a/tmpl/header.tt2 b/tmpl/header.tt2 new file mode 100644 index 0000000..0c5b6f6 --- /dev/null +++ b/tmpl/header.tt2 @@ -0,0 +1,81 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" +"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"> + <head> + <base href="[% base_url %]" /> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + + <script type="text/javascript" src="../../js/lxrng-funcs.js"></script> + <link rel="stylesheet" href="../../css/lxrng.css" type="text/css" title="LXRng" /> + + [% javascript %] + + <script type="text/javascript"> + var use_ajax_navigation=[% IF is_ajax %]1[% ELSE %]0[% END %]; + var use_popup_navigation=[% IF is_popup %]1[% ELSE %]0[% END %]; + </script> + + <title>LXR [% context.tree -%]/ + [%- FOREACH elem = context.path_elements %][% elem.node %][% END %] + </title> + </head> + + <body class="full" + [% IF is_ajax %]onload="load_content();"[% END %] + [% IF is_popup %]onload="popup_prepare([% popup_serial %]);"[% END %]> + + <div class="heading"> + <div class="headingtop"></div> + + <span class="lxr_title"> + <span class="lxr_l">l</span><span class="lxr_x">x</span><span class="lxr_r">r</span> + <span id="current_path"> + <a href="../[% context.vtree %]/">[% context.tree %]/</a>[% + FOREACH elem = context.path_elements + %]<a href="[% elem.path %]">[% elem.node %]</a>[% + END %] + </span> + <img src="../../gfx/rolldown.png" /> + [% IF node.name.match('[^/]$') %] + <form action="+print=[% node.name %]" method="post"> + <button type="submit" class="print"><img src="../../gfx/print.png"></button> + </form> + [% END %] + </span> + + <div class="lxr_menu"> + <span class="lxr_version"> + <img src="../../gfx/left.png" alt="<<"/> + <form action="[% node.name %]" method="get"> + <span id="ver_select"> + [% INCLUDE release_select.tt2, context = context %] + </span> + </form> + <img src="../../gfx/right.png" alt=">>"/> + </span> + + <span class="lxr_search"> + <form action="+search" method="post" onsubmit="return do_search(this);"> + <input type="hidden" name="navtarget" value="" /> + <input type="text" name="search" id="search" /> + <button type="submit">Search</button> + </form> + </span> + + <span class="lxr_prefs"> + <a href="+prefs?return=[% node.name %]" + onclick="return ajax_prefs();"> + Prefs + </a> + </span> + </div> + <form action="+ajax" method="post" onsubmit="return false;"> + <input type="hidden" name="ajax_lookup" id="ajax_lookup" value="" /> + </form> + + <div class="headingbottom"></div> + </div> + + <div id="search_results" class="search_results" + [% IF search_res %]style="display: block;"[% END %]> + [% IF search_res %][% search_res %][% END %]</div> diff --git a/tmpl/line_reference.tt2 b/tmpl/line_reference.tt2 new file mode 100644 index 0000000..f68296c --- /dev/null +++ b/tmpl/line_reference.tt2 @@ -0,0 +1,4 @@ +<a href="[% file %]#L[% line %]" [% navtarget %] +onclick="return load_file('[% context.tree %]', '[% file %][% context.args_url %]', '[% context.release %]', [% line %]);"> + [% file %], line [% line %] +</a> diff --git a/tmpl/main.tt2 b/tmpl/main.tt2 new file mode 100644 index 0000000..0b7ca51 --- /dev/null +++ b/tmpl/main.tt2 @@ -0,0 +1,9 @@ +[% INCLUDE header.tt2 %] +<div id="content"> +[% IF is_dir %] +[% INCLUDE content_dir.tt2 context = context, node = node %] +[% ELSE %] +[% file_content %] +[% END %] +</div> +[% INCLUDE footer.tt2 %] diff --git a/tmpl/popup_main.tt2 b/tmpl/popup_main.tt2 new file mode 100644 index 0000000..5fa87a4 --- /dev/null +++ b/tmpl/popup_main.tt2 @@ -0,0 +1,28 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" +"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"> + <head> + <base href="[% base_url %]" /> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + + <script type="text/javascript" src="../../js/lxrng-funcs.js"></script> + <link rel="stylesheet" href="../../css/lxrng.css" type="text/css" title="LXRng" /> + + <script type="text/javascript"> + var use_ajax_navigation=[% IF is_ajax %]1[% ELSE %]0[% END %]; + var use_popup_navigation=[% IF is_popup %]1[% ELSE %]0[% END %]; + </script> + + <title>LXR Lookup</title> + </head> + + <body class="popup" onload="window.focus();"> + <div id="search_results" class="search_results" style="display: block;"> + [% search_res %] + </div> + </body> +</html> + + + + diff --git a/tmpl/prefs.tt2 b/tmpl/prefs.tt2 new file mode 100644 index 0000000..c7f6a6d --- /dev/null +++ b/tmpl/prefs.tt2 @@ -0,0 +1,59 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" +"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + <script type="text/javascript" src="../../js/lxrng-funcs.js"></script> + <link rel="stylesheet" href="../../css/lxrng.css" type="text/css" title="LXRng" /> + + <title id="title">LXR Preferences</title> + </head> + <body> + + <div class="heading"> + <div class="headingtop"></div> + + <span class="lxr_title"> + <span class="lxr_l">l</span><span class="lxr_x">x</span><span class="lxr_r">r</span> + </span> + + <div class="lxr_menu"> + <span class="lxr_prefs"><a href="[% return || '.' %]">Back</a></span> + </div> + <div class="headingbottom"></div> + </div> + + <div class="pref_heading">Personal preferences for this LXRng site</div> + + <form method="post" action="+prefs"> + <div> + <input type="hidden" name="return" value="[% return %]" /> + Where do you want your search results to be displayed? + <ol> + <li><input type="radio" name="resultloc" value="replace" /> + Replace the active source browsing window</li> + + <li><input type="radio" name="resultloc" value="popup" /> + Show them in a popup window (requires JavaScript enabled)<br> + + <font size="-1"><em>If your browser limits the ability to + raise/lower windows from JavaScript (Firefox: Edit -> + Preferences -> Content -> Enable JavaScript -> Advanced -> + Allow scripts to: Raise or lower windows), make sure you + either close your search result windows or avoid hiding them + behing other windows after use.</em></font></li> + + <li><input type="radio" name="resultloc" value="ajax" /> + Show them inside the active source browsing window + (requires JavaScript enabled) + </li> + </ol> + <p> + (Cookies need to be enabled for LXR preferences to take effect.) + </p> + <button type="submit">Store preferences</button> + </div> + </form> + + </body> +</html> diff --git a/tmpl/prefs_set.tt2 b/tmpl/prefs_set.tt2 new file mode 100644 index 0000000..735e700 --- /dev/null +++ b/tmpl/prefs_set.tt2 @@ -0,0 +1,10 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" +"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"> + <head> + <meta http-equiv="refresh" content="0;url=[% return %]" /> + </head> + <body onload="location.href='[% return %]';"> + <small><a href="[% return %]">Return to the source...</a></small> + </body> +</html> diff --git a/tmpl/print_pdf.tt2 b/tmpl/print_pdf.tt2 new file mode 100644 index 0000000..0577159 --- /dev/null +++ b/tmpl/print_pdf.tt2 @@ -0,0 +1,32 @@ +% -*- latex -*- +\documentclass[a4paper,11pt]{book} +\usepackage[top=1cm, bottom=2cm, left=1.5cm, right=1cm]{geometry} +\usepackage[latin1]{inputenc} +\usepackage[T1]{fontenc} +%% Uncomment the following two lines to produce pdfs with a prettier +%% alternative to the computer modern fonts. +\usepackage{lmodern} +\usepackage{textcomp} +\usepackage{fancyhdr} +\usepackage{bbding} +\pagestyle{fancy} +\begin{document} +\setlength\baselineskip{7pt} +\setlength\parskip{.3\baselineskip} +\setlength\parindent{0pt} +\renewcommand{\headrulewidth}{0pt} +\fancyhead{} +\fancyfoot{} +\def\dash{\raise2.1pt\hbox{\rule{5pt}{0.3pt}}\hspace{1pt}} +\newcommand\lxrln[1]{\tiny\hspace*{-4em}\makebox[4em][r]{#1\hspace{1.5ex}}\small} +\sffamily\small +%% I kinda like the proportional fonts, but expect they won't be +%% everyone's cup of tea. Uncomment next line to use monospace font. +% \tt +\fancyfoot[RE,LO]{\textit{\sffamily [% pathdesc %]% +}} +\fancyfoot[LE,RO]{\textit{\sffamily \thepage}} +[% lineno = 1 %][% FOREACH line = lines -%] +\mbox{}[% line %][% lineno = lineno + 1 %]\\ +[% END %] +\end{document} diff --git a/tmpl/release_select.tt2 b/tmpl/release_select.tt2 new file mode 100644 index 0000000..fc51500 --- /dev/null +++ b/tmpl/release_select.tt2 @@ -0,0 +1,8 @@ +<select name="v" id="ver_list" +onchange="update_version(this, '[% context.base_url %]', '[% context.tree %]', '[% context.default_release %]', '[% context.path %]');"> + [% FOREACH v = context.all_releases %] + <option value="[% v %]" + [% IF v == context.release %]selected="selected"[% END %]> + [% v %]</option> + [% END %] +</select> diff --git a/tmpl/search_result.tt2 b/tmpl/search_result.tt2 new file mode 100644 index 0000000..639e951 --- /dev/null +++ b/tmpl/search_result.tt2 @@ -0,0 +1,74 @@ +<span class="close-button"> +<a href="#" onclick="return [% IF navtarget %]window.close();[% ELSE %]hide_search();[% END %]"> +<img border="0" src="../../gfx/close.png" alt="X"></a> +</span> + +[% IF code_res and code_res.idents %] +<div class="query_desc">Code search: [% code_res.query %]</div> +<ul> +[% FOREACH ident = code_res.idents %] +<li><a class="iref" + href="+ident=[% ident.0 %][% IF navtarget %]?nav[% navtarget %][% END %]" + onclick="return ajax_lookup_anchor(null, this);"> +[% ident.1 %]</a> at [% INCLUDE line_reference.tt2, file = ident.2, line = ident.3 %] +</li> +[% END %] +</ul> +[% END %] + +[% IF ident_res %] +<div class="query_desc">Identifier: +[% ident_res.ident.1 %] +<a class="sref" + href="+code=[% ident_res.query %][% IF navtarget %]?nav[% navtarget %][% END %]" + onclick="return ajax_lookup_anchor(null, this);"> +[% ident_res.query %]</a>, from +[% INCLUDE line_reference.tt2, + file = ident_res.ident.2, line = ident_res.ident.3 %] +</div> +<em>References:</em> +<ul> +[% FOREACH file = ident_res.refs.keys %] +[% FOREACH line = ident_res.refs.$file %] +<li> +[% INCLUDE line_reference.tt2, + file = file, line = line %] +</li> +[% END %] +[% END %] +</ul> +[% END %] + +[% IF text_res %] +<div class="query_desc">Freetext search: [% text_res.query %] +([% text_res.total %] estimated hits)</div> +<ul> +[% FOREACH file = text_res.files %] +<li>[% file.0 %]% +at [% INCLUDE line_reference.tt2, file = file.1, line = file.2 %] +</li> +[% END %] +</ul> +[% END %] + +[% IF file_res %] +<div class="query_desc">Filename search: [% file_res.query %]</div> +<ul> +[% FOREACH file = file_res.files %] +<li><a href="[% file %]" onclick="return load_file('[% context.tree %]', + '[% file %][% context.args_url %]', '', '');" [% navtarget %]>[% file %]</a> +</li> +[% END %] +</ul> +[% END %] + +[% IF ambig_res %] +<div class="query_desc">Ambiguous file reference, please choose one:</div> +<ul> +[% FOREACH file = ambig_res.files %] +<li><a href="[% file %]" onclick="return load_file('[% context.tree %]', + '[% file %][% context.args_url %]', '', '');" [% navtarget %]>[% file %]</a> +</li> +[% END %] +</ul> +[% END %] |
