Compare commits

...

2 Commits

Author SHA1 Message Date
Alex Talreja
2b09b69d2e adjust UI design to use icons and better match style 2025-08-06 17:15:54 +00:00
Alex Talreja
3ebc3d0907 feat: upload yaml and provide source config templates 2025-08-05 23:43:06 +00:00
9 changed files with 837 additions and 0 deletions

View File

@@ -41,6 +41,7 @@ body {
display: flex;
flex-direction: column;
padding: 15px;
padding-bottom: 30px;
align-items: center;
width: 100%;
height: 100%;
@@ -168,6 +169,38 @@ body {
border: 2px solid var(--text-primary-gray);
}
.btn--clear-yaml {
background-color: transparent;
border: 2px solid var(--button-secondary);
border-radius: 5px;
color: var(--button-secondary)
}
.btn--download-yaml {
background-color: var(--button-secondary);
border-radius: 5px;
}
.btn--yaml-tab {
background-color: transparent;
color: var(--text-secondary-gray);
border: 2px solid var(--text-secondary-gray);
border-radius: 10px;
text-decoration: none;
font-size: 1.2em;
font-family: 'Trebuchet MS', 'Lucida Sans Unicode', 'Lucida Grande', 'Lucida Sans', Arial, sans-serif;
font-weight: bold;
width: 100%;
&.active {
background-color: #d0d0d0;
&:hover {
background-color: #d0d0d0;
}
}
}
.tool-button {
display: flex;
align-items: center;
@@ -666,6 +699,166 @@ body {
}
}
}
.editor-container {
display: flex;
flex-direction: row;
flex-grow: 1;
min-height: 0;
width: 100%;
gap: 20px;
}
#yaml-editor-area {
flex: 3;
padding: 20px;
display: flex;
flex-direction: column;
overflow: hidden;
overflow-x: auto;
background-color: #ffffff;
border: 1px solid #ddd;
border-radius: 8px;
}
#source-adder {
flex: 1;
padding: 20px;
border: 1px solid #ddd;
background-color: #f9f9f9;
border-radius: 8px;
overflow-y: auto;
h3 {
margin-top: 0;
margin-bottom: 15px;
color: var(--text-primary-gray);
}
}
#yaml-uploader {
margin-bottom: 15px;
flex-shrink: 0;
}
#yaml-uploader h3 {
margin-top: 0;
margin-bottom: 10px;
color: var(--text-primary-gray);
}
#yaml-uploader p {
margin-top: 0;
margin-bottom: 10px;
font-size: 0.9em;
color: var(--text-secondary-gray);
}
#controls {
display: flex;
justify-content: space-between;
gap: 20px;
align-items: center;
margin-bottom: 10px;
}
.yaml-output-container {
display: flex;
flex-direction: column;
flex-grow: 1;
min-height: 0;
}
.editor-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 5px;
flex-shrink: 0;
h3 {
margin: 0;
color: var(--text-primary-gray);
}
}
.header-actions {
display: flex;
gap: 6px;
}
.icon-button {
background: none;
border: none;
padding: 6px;
cursor: pointer;
color: var(--text-secondary-gray);
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
transition: background-color 0.2s ease, color 0.2s ease;
&:hover {
background-color: #f0f0f0;
color: var(--text-primary-gray);
}
}
.icon-button .material-icons {
font-size: 22px;
}
#yamlOutput {
width: 100%;
flex-grow: 1;
font-family: monospace;
display: block;
padding: 10px;
border: 1px solid #ccc;
border-radius: 4px;
resize: none;
overflow-y: auto;
box-sizing: border-box;
min-height: 200px;
}
#sourceTypeSelect {
width: 100%;
padding: 10px;
margin-top: 10px;
border: 1px solid #ddd;
border-radius: 4px;
}
.hidden-file-input {
width: 0.1px;
height: 0.1px;
opacity: 0;
overflow: hidden;
position: absolute;
z-index: -1;
}
#loadedFileName {
margin-top: 8px;
font-style: italic;
color: #555;
font-size: 0.9em;
word-break: break-all;
}
#fileError {
color: red;
margin-top: 8px;
font-size: 0.9em;
}
.upload-yaml-tab {
padding-top: 15px;
margin-top: auto;
width: 100%;
}
.toggle-details-tab {
background-color: transparent;

View File

@@ -0,0 +1,140 @@
package main
import (
"encoding/json"
"fmt"
"go/ast"
"go/parser"
"go/token"
"log"
"os"
"path/filepath"
"reflect"
"strings"
)
const (
sourcesDir = "internal/sources"
outputFile = "internal/server/static/data/source_templates.json"
)
func main() {
log.SetFlags(0)
results := make(map[string][]string)
absSourcesDir, err := filepath.Abs(sourcesDir)
if err != nil {
log.Fatalf("Failed to get absolute path for sources_dir: %v", err)
}
entries, err := os.ReadDir(absSourcesDir)
if err != nil {
log.Fatalf("Error reading sources directory %s: %v", absSourcesDir, err)
}
for _, entry := range entries {
if entry.IsDir() {
sourceName := entry.Name()
goFilePath := filepath.Join(absSourcesDir, sourceName, sourceName+".go")
if _, err := os.Stat(goFilePath); err == nil {
log.Printf("Processing: %s", goFilePath)
fields, err := parseConfigFields(goFilePath)
if err != nil {
log.Printf("Error parsing %s: %v", goFilePath, err)
continue
}
if len(fields) > 0 {
results[sourceName] = fields
} else {
log.Printf("Warning: No relevant fields found in 'Config' struct in %s", goFilePath)
}
} else if os.IsNotExist(err) {
// This is fine, not all subdirs might follow the pattern
} else {
log.Printf("Error stating file %s: %v", goFilePath, err)
}
}
}
if len(results) == 0 {
log.Printf("No source configs found matching the pattern in %s", absSourcesDir)
}
jsonData, err := json.MarshalIndent(results, "", " ")
if err != nil {
log.Fatalf("Error marshaling JSON: %v", err)
}
absOutputFile, err := filepath.Abs(outputFile)
if err != nil {
log.Fatalf("Failed to get absolute path for output_file: %v", err)
}
// Ensure the output directory exists
outputDir := filepath.Dir(absOutputFile)
if err := os.MkdirAll(outputDir, 0755); err != nil {
log.Fatalf("Error creating output directory %s: %v", outputDir, err)
}
err = os.WriteFile(absOutputFile, jsonData, 0644)
if err != nil {
log.Fatalf("Error writing output file %s: %v", absOutputFile, err)
}
fmt.Printf("Successfully generated %s\n", absOutputFile)
}
func parseConfigFields(filename string) ([]string, error) {
fset := token.NewFileSet()
node, err := parser.ParseFile(fset, filename, nil, 0)
if err != nil {
return nil, err
}
var fields []string
foundConfig := false
ast.Inspect(node, func(n ast.Node) bool {
if foundConfig {
return false
}
typeSpec, ok := n.(*ast.TypeSpec)
if !ok || typeSpec.Name.Name != "Config" {
return true
}
structType, ok := typeSpec.Type.(*ast.StructType)
if !ok {
return true
}
foundConfig = true // Mark as found
for _, field := range structType.Fields.List {
if len(field.Names) == 0 {
continue
}
fieldName := field.Names[0].Name
if fieldName == "Name" {
continue
}
yamlTagName := ""
if field.Tag != nil {
tagVal := strings.Trim(field.Tag.Value, "`")
tags := reflect.StructTag(tagVal)
if yamlTag, ok := tags.Lookup("yaml"); ok {
yamlTagName = strings.Split(yamlTag, ",")[0]
}
}
if yamlTagName != "" && yamlTagName != "-" {
fields = append(fields, yamlTagName)
}
}
return false
})
return fields, nil
}

View File

@@ -0,0 +1,132 @@
{
"bigquery": [
"kind",
"project",
"location"
],
"bigtable": [
"kind",
"project",
"instance"
],
"couchbase": [
"kind",
"connectionString",
"bucket",
"scope",
"username",
"password",
"clientCert",
"clientCertPassword",
"clientKey",
"clientKeyPassword",
"caCert",
"noSslVerify",
"profile",
"queryScanConsistency"
],
"dataplex": [
"kind",
"project"
],
"dgraph": [
"kind",
"dgraphUrl",
"user",
"password",
"namespace",
"apiKey"
],
"duckdb": [
"kind",
"dbFilePath",
"configuration"
],
"firestore": [
"kind",
"project",
"database"
],
"http": [
"kind",
"baseUrl",
"timeout",
"headers",
"queryParams",
"disableSslVerification"
],
"looker": [
"kind",
"base_url",
"client_id",
"client_secret",
"verify_ssl",
"timeout"
],
"mongodb": [
"kind",
"uri"
],
"mssql": [
"kind",
"host",
"port",
"user",
"password",
"database",
"encrypt"
],
"mysql": [
"kind",
"host",
"port",
"user",
"password",
"database",
"queryTimeout"
],
"neo4j": [
"kind",
"uri",
"user",
"password",
"database"
],
"postgres": [
"kind",
"host",
"port",
"user",
"password",
"database"
],
"redis": [
"kind",
"address",
"username",
"password",
"database",
"useGCPIAM",
"clusterEnabled"
],
"spanner": [
"kind",
"project",
"instance",
"dialect",
"database"
],
"sqlite": [
"kind",
"database"
],
"valkey": [
"kind",
"address",
"username",
"password",
"database",
"useGCPIAM",
"disableCache"
]
}

View File

@@ -35,6 +35,9 @@ function renderNavbar(containerId, activePath) {
<li><a href="/ui/tools">Tools</a></li>
<li><a href="/ui/toolsets">Toolsets</a></li>
</ul>
<div class="upload-yaml-tab">
<a href="/ui/upload" class="btn btn--yaml-tab">YAML Builder</a>
</div>
</nav>
`;
@@ -49,5 +52,15 @@ function renderNavbar(containerId, activePath) {
link.classList.remove('active');
}
});
const yamlButton = navbarContainer.querySelector('.btn--yaml-tab');
if (yamlButton) {
const buttonPath = new URL(yamlButton.href).pathname;
if (buttonPath === activePath) {
yamlButton.classList.add('active');
} else {
yamlButton.classList.remove('active');
}
}
}
}

View File

@@ -0,0 +1,91 @@
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
document.addEventListener('DOMContentLoaded', () => {
const sourceTypeSelect = document.getElementById('sourceTypeSelect');
const yamlOutputTextarea = document.getElementById('yamlOutput');
let dbConfigData = {};
const jsonPath = '/ui/data/source_templates.json';
async function loadDbConfigs() {
try {
const response = await fetch(jsonPath);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status} fetching ${jsonPath}`);
}
dbConfigData = await response.json();
populateDropdown();
} catch (error) {
console.error('Could not load database types:', error);
sourceTypeSelect.disabled = true;
let errorOption = sourceTypeSelect.querySelector('option[value=""]');
if (errorOption) {
errorOption.textContent = 'Failed to load types';
}
}
}
function populateDropdown() {
if (!dbConfigData || Object.keys(dbConfigData).length === 0) {
console.warn('No database types found in JSON data.');
return;
}
Object.keys(dbConfigData).sort().forEach(dbType => {
const option = document.createElement('option');
option.value = dbType;
option.textContent = dbType;
sourceTypeSelect.appendChild(option);
});
}
sourceTypeSelect.addEventListener('change', () => {
const selectedType = sourceTypeSelect.value;
if (!selectedType || !dbConfigData[selectedType]) {
return;
}
const fields = dbConfigData[selectedType];
if (!Array.isArray(fields)) {
console.error(`Fields for ${selectedType} is not an array.`);
return;
}
const currentYaml = yamlOutputTextarea.value;
const lines = currentYaml.split('\n');
const sourcesIndex = lines.findIndex(line => line.trim() === 'sources:');
if (sourcesIndex === -1) {
alert('The line "sources:" was not found in the YAML content. Please add it to insert a source.');
sourceTypeSelect.value = "";
return;
}
const NAME_INDENT = ' ';
const FIELD_INDENT = ' ';
let snippetLines = [];
snippetLines.push(`${NAME_INDENT}YOUR_${selectedType.toUpperCase()}_SOURCE_NAME:`);
fields.forEach(field => {
snippetLines.push(`${FIELD_INDENT}${field}:`);
});
lines.splice(sourcesIndex + 1, 0, ...snippetLines);
yamlOutputTextarea.value = lines.join('\n');
yamlOutputTextarea.scrollTop = 0;
sourceTypeSelect.value = "";
});
loadDbConfigs();
});

View File

@@ -0,0 +1,184 @@
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
document.addEventListener('DOMContentLoaded', () => {
const SESSION_STORAGE_KEY = 'yamlUploaderSessionData';
const yamlFileInput = document.getElementById('yamlFileInput');
const fileError = document.getElementById('fileError');
const yamlOutput = document.getElementById('yamlOutput');
const clearSessionButton = document.getElementById('clearSessionButton');
const loadedFileNameDisplay = document.getElementById('loadedFileName');
const downloadYamlButton = document.getElementById('downloadYamlButton');
function debounce(func, delay) {
let timeoutId;
return (...args) => {
clearTimeout(timeoutId);
timeoutId = setTimeout(() => {
func.apply(this, args);
}, delay);
};
}
function saveToSessionStorage(filename, yamlContent) {
try {
const data = { filename: filename, yamlContent: yamlContent };
sessionStorage.setItem(SESSION_STORAGE_KEY, JSON.stringify(data));
console.debug('Saved to sessionStorage');
} catch (e) {
console.error("Error saving to sessionStorage:", e);
fileError.textContent = "Failed to save data to session storage.";
}
}
function loadFromSessionStorage() {
try {
const storedData = sessionStorage.getItem(SESSION_STORAGE_KEY);
if (storedData) {
const data = JSON.parse(storedData);
if (data.yamlContent) {
yamlOutput.value = data.yamlContent;
}
if (data.filename) {
loadedFileNameDisplay.textContent = `Current file: ${data.filename}`;
} else {
loadedFileNameDisplay.textContent = "";
}
console.debug('Loaded from sessionStorage');
} else {
console.debug('No data in sessionStorage');
}
} catch (e) {
console.error("Error loading from sessionStorage:", e);
fileError.textContent = "Failed to load data from session storage.";
sessionStorage.removeItem(SESSION_STORAGE_KEY);
}
}
function clearSessionStorage() {
sessionStorage.removeItem(SESSION_STORAGE_KEY);
yamlOutput.value = '';
fileError.textContent = '';
yamlFileInput.value = '';
loadedFileNameDisplay.textContent = '';
console.log("Cleared stored YAML from sessionStorage.");
}
if (yamlFileInput) {
yamlFileInput.addEventListener('change', (event) => {
const file = event.target.files[0];
if (!file) {
console.log('No file selected.');
return;
}
const fileName = file.name;
if (!fileName.toLowerCase().endsWith('.yaml') && !fileName.toLowerCase().endsWith('.yml')) {
fileError.textContent = 'Invalid file type. Please upload a .yaml or .yml file.';
yamlFileInput.value = '';
return;
}
fileError.textContent = '';
const reader = new FileReader();
reader.onload = (e) => {
const content = e.target.result;
try {
if (typeof jsyaml === 'undefined') {
fileError.textContent = 'Error: js-yaml library not loaded.';
return;
}
const parsedYaml = jsyaml.load(content);
const prettyYaml = jsyaml.dump(parsedYaml, { indent: 2, lineWidth: 120, noRefs: true });
yamlOutput.value = prettyYaml;
loadedFileNameDisplay.textContent = `Current file: ${fileName}`;
saveToSessionStorage(fileName, prettyYaml);
} catch (err) {
fileError.textContent = 'Error parsing YAML: ' + err.message;
console.error('YAML Parsing Error:', err);
yamlOutput.value = '';
loadedFileNameDisplay.textContent = '';
sessionStorage.removeItem(SESSION_STORAGE_KEY);
}
};
reader.onerror = () => {
fileError.textContent = 'Error reading file.';
};
reader.readAsText(file);
});
}
if (yamlOutput) {
yamlOutput.addEventListener('input', debounce(() => {
const currentFilename = getDownloadFilename();
saveToSessionStorage(currentFilename, yamlOutput.value);
console.debug("YAML content changes saved to sessionStorage.");
}, 500));
}
if (clearSessionButton) {
clearSessionButton.addEventListener('click', () => {
if (confirm("Clear all content from the YAML editor? This cannot be undone.")) {
clearSessionStorage();
console.debug("User confirmed clearing session.");
} else {
console.debug("User cancelled clearing session.");
}
});
}
function getDownloadFilename() {
const loadedFileName = loadedFileNameDisplay.textContent;
let baseName = 'tools';
if (loadedFileName && loadedFileName.startsWith('Current file: ')) {
const originalFileName = loadedFileName.substring('Current file: '.length);
baseName = originalFileName.replace(/\.ya?ml$/i, '');
}
return `${baseName}_MODIFIED.yaml`;
}
if (downloadYamlButton) {
downloadYamlButton.addEventListener('click', () => {
const yamlContent = yamlOutput.value;
if (!yamlContent) {
alert("Text area is empty. Nothing to download.");
return;
}
const fileName = getDownloadFilename();
const blob = new Blob([yamlContent], { type: 'application/octet-stream' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.style.display = 'none';
a.href = url;
a.download = fileName;
document.body.appendChild(a);
a.click();
setTimeout(() => {
document.body.removeChild(a);
URL.revokeObjectURL(url);
console.debug('Download link cleaned up');
}, 100);
});
}
loadFromSessionStorage();
});

View File

@@ -0,0 +1,69 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>YAML Builder View</title>
<link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
<link rel="stylesheet" href="/ui/css/style.css">
<script src="https://cdn.jsdelivr.net/npm/js-yaml@4.1.0/dist/js-yaml.min.js"></script>
</head>
<body>
<div id="navbar-container" data-active-nav="/ui/upload"></div>
<div id="main-content-container">
<div class="editor-container">
<div id="yaml-editor-area">
<div id="yaml-uploader">
<h3>Upload YAML File</h3>
<p>Please select a file ending with .yaml or .yml</p>
<div id="controls">
<div>
<input type="file" id="yamlFileInput" accept=".yaml,.yml" class="hidden-file-input">
<label for="yamlFileInput" class="btn btn--run">Choose File</label>
</div>
</div>
<div id="fileError"></div>
<div id="loadedFileName"></div>
</div>
<div class="yaml-output-container">
<div class="editor-header">
<h3>Formatted YAML Output:</h3>
<div class="header-actions">
<button id="clearSessionButton" class="icon-button" title="Clear YAML">
<i class="material-icons">delete</i>
</button>
<button id="downloadYamlButton" class="icon-button" title="Download YAML">
<i class="material-icons">file_download</i>
</button>
</div>
</div>
<textarea id="yamlOutput" placeholder="Upload a YAML file or paste content here..."></textarea>
</div>
</div>
<div id="source-adder">
<h3>Add Source Snippet</h3>
<select id="sourceTypeSelect">
<option value="">Select DB Type...</option>
</select>
<p style="font-size: 0.8em; color: #666;">Select a database type to append a template to the YAML output.</p>
</div>
</div>
</div>
<script src="/ui/js/navbar.js"></script>
<script src="/ui/js/uploadYaml.js"></script>
<script src="/ui/js/sourceSelector.js"></script>
<script>
document.addEventListener('DOMContentLoaded', () => {
const navbarContainer = document.getElementById('navbar-container');
const activeNav = navbarContainer.getAttribute('data-active-nav');
if (typeof renderNavbar === 'function') {
renderNavbar('navbar-container', activeNav);
} else {
console.error('renderNavbar function not found.');
}
});
</script>
</body>
</html>

View File

@@ -24,6 +24,7 @@ func webRouter() (chi.Router, error) {
r.Get("/", func(w http.ResponseWriter, r *http.Request) { serveHTML(w, r, "static/index.html") })
r.Get("/tools", func(w http.ResponseWriter, r *http.Request) { serveHTML(w, r, "static/tools.html") })
r.Get("/toolsets", func(w http.ResponseWriter, r *http.Request) { serveHTML(w, r, "static/toolsets.html") })
r.Get("/upload", func(w http.ResponseWriter, r *http.Request) { serveHTML(w, r, "static/uploadYaml.html") })
// handler for all other static files/assets
staticFS, _ := fs.Sub(staticContent, "static")

View File

@@ -73,6 +73,20 @@ func TestWebEndpoint(t *testing.T) {
wantContentType: "text/html",
wantPageTitle: "Toolsets View",
},
{
name: "web yaml builder page",
path: "/ui/upload",
wantStatus: http.StatusOK,
wantContentType: "text/html",
wantPageTitle: "YAML Builder View",
},
{
name: "web yaml builder page with trailing slash",
path: "/ui/upload/",
wantStatus: http.StatusOK,
wantContentType: "text/html",
wantPageTitle: "YAML Builder View",
},
}
for _, tc := range testCases {