helpers.go

package main

// helpers module
//
// Copyright (c) 2023 - Valentin Kuznetsov <vkuznet@gmail.com>
//
import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"

	beamlines "github.com/CHESSComputing/golib/beamlines"
	srvConfig "github.com/CHESSComputing/golib/config"
	schema "github.com/CHESSComputing/golib/schema"
	server "github.com/CHESSComputing/golib/server"
	services "github.com/CHESSComputing/golib/services"
	utils "github.com/CHESSComputing/golib/utils"
	"github.com/gin-gonic/gin"
)

//
// helper functions
//

// helper function to provide error page
func handleError(c *gin.Context, code int, msg string, err error) {
	page := server.ErrorPage(StaticFs, msg, err)
	if c.Request.Header.Get("Accept") == "application/json" {
		c.JSON(code, gin.H{"error": err.Error(), "message": msg, "code": code})
		return
	}
	c.Data(code, "text/html; charset=utf-8", []byte(header()+page+footer()))
}

// helper function to provides error template message
func errorTmpl(c *gin.Context, msg string, err error) string {
	tmpl := server.MakeTmpl(StaticFs, "Status")
	tmpl["Content"] = template.HTML(fmt.Sprintf("<div>%s</div>\n<br/><h3>ERROR</h3>%v", msg, err))
	content := server.TmplPage(StaticFs, "error.tmpl", tmpl)
	return content
}

// helper functiont to provides success template message
func successTmpl(c *gin.Context, msg string) string {
	tmpl := server.MakeTmpl(StaticFs, "Status")
	tmpl["Content"] = template.HTML(fmt.Sprintf("<h3>SUCCESS</h3><div>%s</div>", msg))
	content := server.TmplPage(StaticFs, "success.tmpl", tmpl)
	return content
}

// helper funtion to get record value
func recValue(rec map[string]any, attr string) string {
	if val, ok := rec[attr]; ok {
		switch v := val.(type) {
		case float64:
			if attr == "did" {
				return fmt.Sprintf("%d", int64(val.(float64)))
			}
			return fmt.Sprintf("%f", v)
		default:
			return fmt.Sprintf("%v", v)
		}
	}
	return "Not available"
}

// helper function to get dids from DataHub service
func datahubDidHashes() []string {
	var didHashes []string
	if srvConfig.Config.DataHubURL == "" {
		return didHashes
	}
	_httpReadRequest.GetToken()
	rurl := fmt.Sprintf("%s/datahub", srvConfig.Config.DataHubURL)
	resp, err := _httpReadRequest.Get(rurl)
	if err != nil {
		log.Println("WARNING: unable to get datahub didHashes, error:", err)
		return didHashes
	}
	defer resp.Body.Close()
	data, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Println("WARNING: unable to get datahub didHashes, error:", err)
		return didHashes
	}
	var arr []string
	err = json.Unmarshal(data, &arr)
	if err != nil {
		log.Println("WARNING: unable to get datahub didHashes, error:", err)
		return didHashes
	}
	for _, entry := range arr {
		if did, err := url.QueryUnescape(entry); err == nil {
			didHashes = append(didHashes, did)
		}
	}
	return didHashes
}

// helper function to prepare HTML page for given services records
func records2html(user string, records []map[string]any, attrs2show []string) string {
	var out []string
	didhashes := datahubDidHashes()
	for _, rec := range records {
		tmpl := server.MakeTmpl(StaticFs, "Record")
		tmpl["User"] = user
		tmpl["Id"] = recValue(rec, "did")
		did := recValue(rec, "did")
		tmpl["Did"] = did
		if val, err := url.QueryUnescape(did); err == nil {
			tmpl["DidEncoded"] = val
		}
		tmpl["Cycle"] = recValue(rec, "cycle")
		tmpl["Beamline"] = recValue(rec, "beamline")
		btr := recValue(rec, "btr")
		tmpl["Btr"] = btr
		project := recValue(rec, "project")
		if project == "Not available" {
			btr := recValue(rec, "btr")
			if btr == "" {
				project = "foxden"
			} else {
				project = btr
			}
		}
		if !strings.HasPrefix(project, "chess_") {
			project = fmt.Sprintf("chess_%s", project)
		}
		tmpl["MCProjectName"] = project
		tmpl["Sample"] = recValue(rec, "sample_name")
		tmpl["Schema"] = recValue(rec, "schema")
		tmpl["Base"] = srvConfig.Config.Frontend.WebServer.Base
		tmpl["Record"] = rec
		tmpl["RecordTable"] = reprRecord(rec, "table")
		tmpl["RecordDescription"] = reprRecord(rec, "description")
		tmpl["RecordJSON"] = reprRecord(rec, "json")
		tmpl["Description"] = recValue(rec, "description")
		if val, err := lastModified(rec); err == nil {
			tmpl["TimeStamp"] = val
		} else {
			tmpl["TimeStamp"] = "Not Available"
		}
		if val, ok := rec["globus_link"]; ok {
			tmpl["GlobusLink"] = fmt.Sprintf("%v", val)
		}
		if val, ok := rec["doi"]; ok {
			tmpl["Doi"] = val
		}
		if val, ok := rec["schema"]; ok {
			tmpl["Schema"] = val
		}
		if val, ok := rec["user"]; ok {
			tmpl["User"] = val
		}
		// first check if there is doi_url
		if val, ok := rec["doi_url"]; ok {
			tmpl["DoiLink"] = val
		}
		// then, check if we created foxden_doi_url and use it instead for web UI link
		if val, ok := rec["doi_foxden_url"]; ok {
			tmpl["DoiLink"] = val
		}
		if val, ok := rec["doi_public"]; ok {
			tmpl["DoiPublic"] = val
		}
		if val, ok := rec["doi_provider"]; ok {
			tmpl["DoiProvider"] = val
		}
		if val, ok := rec["beamline"]; ok {
			var beamlines []string
			switch vvv := val.(type) {
			case []any:
				for _, v := range vvv {
					beamlines = append(beamlines, fmt.Sprintf("%v", v))
				}
			case string:
				beamlines = append(beamlines, vvv)
			case any:
				beamlines = append(beamlines, fmt.Sprintf("%v", vvv))
			}
			for _, b := range beamlines {
				if utils.InList(b, srvConfig.Config.CHESSMetaData.SpecScanBeamlines) {
					tmpl["SpecScanLink"] = fmt.Sprintf("/specscans?did=%s", url.QueryEscape(recValue(rec, "did")))
					break
				}
			}
		}

		// look for data location attributes, if found create Data Management link
		/*
			for _, loc := range srvConfig.Config.CHESSMetaData.DataLocationAttributes {
				if _, ok := rec[loc]; ok {
					tmpl["RawDataLink"] = fmt.Sprintf("/dm?did=%s&attr=data_location_raw", recValue(rec, "did"))
					break
				}
			}
		*/
		if val, ok := rec["data_location_raw"]; ok && val != "" {
			tmpl["RawDataLink"] = fmt.Sprintf("/dm?did=%s&attr=data_location_raw", recValue(rec, "did"))
		}
		if val, ok := rec["data_location_reduced"]; ok && val != "" {
			tmpl["ReducedDataLink"] = fmt.Sprintf("/dm?did=%s&attr=data_location_reduced", recValue(rec, "did"))
		}

		// check if did hash exists in DataHub
		sum := md5.Sum([]byte(did))
		didhash := hex.EncodeToString(sum[:])
		if utils.InList(didhash, didhashes) {
			tmpl["AuxDataLink"] = fmt.Sprintf("%s/datahub/%s", srvConfig.Config.DataHubURL, didhash)
		}

		if val, ok := rec["history"]; ok {
			switch t := val.(type) {
			case []any:
				tmpl["RecordVersion"] = len(t) + 1 // human counter, i.e. if one history record it is 2nd version
			}
		}
		amap := make(map[string]any)
		for _, attr := range attrs2show {
			if val, ok := rec[attr]; ok {
				amap[attr] = val
			}
		}
		tmpl["AttributesMap"] = amap

		if srvConfig.Config.UserMetaDataURL != "" {
			tmpl["UserMetaDataLink"] = fmt.Sprintf("%s/record?did=%s", srvConfig.Config.UserMetaDataURL, did)
		}

		content := server.TmplPage(StaticFs, "record.tmpl", tmpl)
		out = append(out, content)
	}
	return strings.Join(out, "\n")
}

var _metaManager *schema.MetaDataManager

// helper function to represent record
func reprRecord(rec map[string]any, format string) string {
	sname := recValue(rec, "schema")
	umap := _metaManager.Units(sname)
	dmap := _metaManager.Descriptions(sname)
	if format == "json" {
		var srec string
		data, err := json.MarshalIndent(rec, "", "  ")
		if err != nil {
			log.Println("ERROR: unable to marshal record", rec, err)
			srec = "Not available"
		} else {
			srec = string(data)
		}
		return srec
	}
	keys := utils.MapKeys(rec)
	sort.Strings(keys)
	var maxLen int
	for _, k := range keys {
		if len(k) > maxLen {
			maxLen = len(k)
		}
	}
	var out string
	if format == "description" {
		for _, key := range keys {
			if desc, ok := dmap[key]; ok {
				out = fmt.Sprintf("%s\n%s: %v", out, utils.PaddedKey(key, maxLen), desc)
			} else {
				out = fmt.Sprintf("%s\n%s: Not Available", out, utils.PaddedKey(key, maxLen))
			}
		}
		return out
	}
	for _, key := range keys {
		val, _ := rec[key]
		if unit, ok := umap[key]; ok {
			if unit != "" {
				out = fmt.Sprintf("%s\n%s: %v (%s)", out, utils.PaddedKey(key, maxLen), val, unit)
			} else {
				out = fmt.Sprintf("%s\n%s: %v", out, utils.PaddedKey(key, maxLen), val)
			}
		} else {
			out = fmt.Sprintf("%s\n%s: %v", out, utils.PaddedKey(key, maxLen), val)
		}
	}
	return out
}

// helper function to make pagination
func pagination(c *gin.Context, query string, nres, startIdx, limit int, sortKey, sortOrder string, btrs []string) string {
	tmpl := server.MakeTmpl(StaticFs, "Search")
	if user, err := getUser(c); err == nil {
		tmpl["User"] = user
		tmpl["DataAttributes"] = strings.Join(_foxdenAttrs, ",")
	}
	eQuery := url.QueryEscape(query)
	url := fmt.Sprintf("/search?query=%s&sort_keys=%s&sort_order=%s", eQuery, sortKey, sortOrder)
	if nres > 0 {
		tmpl["StartIndex"] = fmt.Sprintf("%d", startIdx+1)
	} else {
		tmpl["StartIndex"] = fmt.Sprintf("%d", startIdx)
	}
	if nres > startIdx+limit {
		tmpl["EndIndex"] = fmt.Sprintf("%d", startIdx+limit)
	} else {
		tmpl["EndIndex"] = fmt.Sprintf("%d", nres)
	}
	tmpl["Total"] = fmt.Sprintf("%d", nres)
	tmpl["FirstUrl"] = makeURL(url, "first", startIdx, limit, nres)
	tmpl["PrevUrl"] = makeURL(url, "prev", startIdx, limit, nres)
	tmpl["NextUrl"] = makeURL(url, "next", startIdx, limit, nres)
	tmpl["LastUrl"] = makeURL(url, "last", startIdx, limit, nres)
	tmpl["Query"] = template.HTML(query)
	tmpl["SortKey"] = sortKey
	tmpl["SortOrder"] = sortOrder
	tmpl["Query"] = query
	tmpl["Btrs"] = btrs
	page := server.TmplPage(StaticFs, "pagination.tmpl", tmpl)
	return fmt.Sprintf("%s", page)
}

// helper function to make URL
func makeURL(url, urlType string, startIdx, limit, nres int) string {
	if limit < 0 {
		limit = nres
	}
	var out string
	var idx int
	if urlType == "first" {
		idx = 0
	} else if urlType == "prev" {
		if startIdx != 0 {
			idx = startIdx - limit
		} else {
			idx = 0
		}
	} else if urlType == "next" {
		idx = startIdx + limit
	} else if urlType == "last" {
		j := 0
		for i := 0; i < nres; i = i + limit {
			if i > nres {
				break
			}
			j = i
		}
		idx = j
	}
	out = fmt.Sprintf("%s&amp;idx=%d&&amp;limit=%d", url, idx, limit)
	return out
}

// helper function to find beameline web UI order from
// FOXDEN configuration. By default we'll use default order (alphabetical)
func findSchemaOrder(schemaFileName string) bool {
	for _, s := range srvConfig.Config.SchemaOrders {
		if s.Schema == schemaFileName || strings.Contains(schemaFileName, s.Schema) {
			return s.Order
		}
	}
	// if order is false we'll use default order (alphabetical)
	return false
}

// helper function to provide beamline section keys
func beamlineSectionKeys(
	schema *beamlines.Schema,
	section string,
	sectionKeys map[string][]string) []string {

	schemaFileName := schema.Name()
	ordered := findSchemaOrder(schemaFileName)
	if !ordered {
		skeys, _ := sectionKeys[section]
		return skeys
	}

	var bskeys []string
	beamlineSections := srvConfig.Config.BeamlineSections
	if len(beamlineSections) == 0 {
		beamlineSections = schema.ConfigSections
	}
	for _, s := range beamlineSections {
		if s.Schema == schemaFileName || strings.Contains(schemaFileName, s.Schema) {
			for _, item := range s.Sections {
				if item.Section == section {
					bskeys = item.Attributes
				}
			}
			break
		}
	}
	if skeys, ok := sectionKeys[section]; ok {
		for _, key := range skeys {
			if !utils.InList(key, bskeys) {
				bskeys = append(bskeys, key)
			}
		}
	}
	return bskeys
}

// helper function to extract beamline sections from FOXDEN configuration
func beamlineSections(schema *beamlines.Schema, sections []string) []string {
	schemaFileName := schema.Name()
	ordered := findSchemaOrder(schemaFileName)
	if !ordered {
		return sections
	}
	var orderedSections []string
	if len(srvConfig.Config.BeamlineSections) == 0 {
		for _, bs := range schema.ConfigSections {
			for _, ws := range bs.Sections {
				orderedSections = append(orderedSections, ws.Section)
			}
		}
		return orderedSections
	}
	var beamlineSections []srvConfig.WebUISection
	for _, s := range srvConfig.Config.BeamlineSections {
		if s.Schema == schemaFileName || strings.Contains(schemaFileName, s.Schema) {
			beamlineSections = s.Sections
			break
		}
	}
	for _, s := range beamlineSections {
		orderedSections = append(orderedSections, s.Section)
	}
	for _, s := range sections {
		if !utils.InList(s, orderedSections) {
			orderedSections = append(orderedSections, s)
		}
	}
	return orderedSections
}

// helper function to generate input form
func genForm(fname string, record *map[string]any) (string, error) {
	var out []string
	val := "<h3>Web form submission</h3><br/>"
	out = append(out, val)
	beamline := utils.FileName(fname)
	if strings.Contains(fname, "user") {
		tmpl := server.MakeTmpl(StaticFs, "Form")
		form := server.TmplPage(StaticFs, "form_user.tmpl", tmpl)
		tmpl["Base"] = srvConfig.Config.Frontend.WebServer.Base
		tmpl["Beamline"] = beamline
		tmpl["Description"] = ""
		tmpl["Form"] = template.HTML(form)
		if record != nil {
			if val, ok := (*record)["description"]; ok {
				tmpl["Description"] = val
			}
		}
		return server.TmplPage(StaticFs, "form_beamline.tmpl", tmpl), nil
	}
	if strings.Contains(strings.ToLower(fname), "composite") {
		tmpl := server.MakeTmpl(StaticFs, "Form")
		form := server.TmplPage(StaticFs, "composite.tmpl", tmpl)
		tmpl["Base"] = srvConfig.Config.Frontend.WebServer.Base
		tmpl["Beamline"] = beamline
		tmpl["Description"] = ""
		if record != nil {
			if val, ok := (*record)["description"]; ok {
				tmpl["Description"] = val
			}
		}
		tmpl["Form"] = template.HTML(form)
		return server.TmplPage(StaticFs, "form_beamline.tmpl", tmpl), nil
	}
	val = fmt.Sprintf("<input class=\"input\" name=\"beamline\" type=\"hidden\" value=\"\"/>%s", beamline)
	schema, err := _smgr.Load(fname)
	if err != nil {
		log.Println("unable to load", fname, "error", err)
		return strings.Join(out, ""), fmt.Errorf("[Frontend.main.genForm] _smgr.Load error: %w", err)
	}
	optKeys, err := schema.OptionalKeys()
	if err != nil {
		log.Println("unable to get optional keys, error", err)
		return strings.Join(out, ""), fmt.Errorf("[Frontend.main.genForm] schema.OptionalKeys error: %w", err)
	}
	allKeys, err := schema.Keys()
	if err != nil {
		log.Println("unable to get keys, error", err)
		return strings.Join(out, ""), fmt.Errorf("[Frontend.main.genForm] schema.Keys error: %w", err)
	}
	sectionKeys, err := schema.SectionKeys()
	if err != nil {
		log.Println("unable to get section keys, error", err)
		return strings.Join(out, ""), fmt.Errorf("[Frontend.main.genForm] schema.SectionKeys error: %w", err)
	}

	// loop over all defined sections
	var rec string
	sections, err := schema.Sections()
	if err != nil {
		log.Println("unable to get sections, error", err)
		return strings.Join(out, ""), fmt.Errorf("[Frontend.main.genForm] schema.Sections error: %w", err)
	}

	for _, s := range beamlineSections(schema, sections) {
		skeys := beamlineSectionKeys(schema, s, sectionKeys)
		if len(skeys) > 0 {
			//if skeys, ok := sectionKeys[s]; ok {
			showSection := false
			if len(skeys) != 0 {
				showSection = true
			}
			// skip sub-records
			subKey := false
			for _, k := range skeys {
				if strings.Contains(k, ".") {
					subKey = true
					break
				}
			}
			if subKey {
				continue
			}
			if showSection {
				out = append(out, fmt.Sprintf("<fieldset id=\"%s\">", s))
				out = append(out, fmt.Sprintf("<legend>%s</legend>", s))
			}
			for _, k := range skeys {
				required := true
				if utils.InList(k, optKeys) {
					required = false
				}
				rec = formEntry(&schema.Map, k, s, required, record)
				out = append(out, rec)
			}
			if showSection {
				out = append(out, "</fieldset>")
			}
		}
	}
	// loop over the rest of section keys which did not show up in sections
	for s, skeys := range sectionKeys {
		if utils.InList(s, sections) {
			continue
		}
		// skip sub-records
		subKey := false
		for _, k := range skeys {
			if strings.Contains(k, ".") {
				subKey = true
				break
			}
		}
		if subKey {
			continue
		}
		showSection := false
		if len(skeys) != 0 {
			showSection = true
		}
		if showSection {
			out = append(out, fmt.Sprintf("<fieldset id=\"%s\">", s))
			out = append(out, fmt.Sprintf("<legend>%s</legend>", s))
		}
		for _, k := range skeys {
			required := true
			if utils.InList(k, optKeys) {
				required = false
			}
			rec = formEntry(&schema.Map, k, s, required, record)
			out = append(out, rec)
		}
		if showSection {
			out = append(out, "</fieldset>")
		}
	}
	// loop over all keys which do not have sections
	var nOut, rOut []string
	legend := "Attributes"
	for _, k := range allKeys {
		if r, ok := schema.Map[k]; ok {
			required := true
			if utils.InList(k, optKeys) {
				required = false
			}
			if strings.Contains(k, ".") {
				subKey := r.Key
				fname := r.File
				rec = formStructEntry(fname, k, subKey)
				section := r.Section
				if legend == "Attributes" {
					legend = section
				}
				rOut = append(rOut, rec)
			} else if r.Section == "" {
				rec = formEntry(&schema.Map, k, "", required, record)
				nOut = append(nOut, rec)
			}
		}
	}
	if len(rOut) > 0 {
		// we need to wrap sub records in repeatable container
		tmpl := server.MakeTmpl(StaticFs, "FormStructEntry")
		tmpl["Section"] = legend
		tmpl["SubRecords"] = strings.Join(rOut, "\n<br/>\n")
		structSection := server.TmplPage(StaticFs, "form_struct.tmpl", tmpl)
		out = append(out, structSection)
	}
	if len(nOut) > 0 {
		nOut = utils.List2Set(nOut)
		out = append(out, "<fieldset id=\"attributes\">")
		out = append(out, fmt.Sprintf("<legend>%s</legend>", legend))
		out = append(out, nOut...)
		out = append(out, "</fieldset>")
	}
	form := strings.Join(out, "\n")
	tmpl := server.MakeTmpl(StaticFs, "Form")
	tmpl["Base"] = srvConfig.Config.Frontend.WebServer.Base
	tmpl["Description"] = ""
	tmpl["Beamline"] = beamline
	if record != nil {
		if val, ok := (*record)["description"]; ok {
			tmpl["Description"] = val
		}
	}
	tmpl["Form"] = template.HTML(form)
	return server.TmplPage(StaticFs, "form_beamline.tmpl", tmpl), nil
}

// helper function to create form entry
func formEntry(
	smap *map[string]beamlines.SchemaRecord,
	skey, section string, required bool, record *map[string]any) string {

	// check if provided record has value
	var defaultValue string
	if record != nil {
		rmap := *record
		if v, ok := rmap[skey]; ok {
			defaultValue = fmt.Sprintf("%v", v)
		}
		defaultValue = strings.ReplaceAll(defaultValue, "[", "")
		defaultValue = strings.ReplaceAll(defaultValue, "]", "")
	}
	tmpl := server.MakeTmpl(StaticFs, "FormEntry")
	tmpl["Key"] = skey
	tmpl["Value"] = defaultValue
	tmpl["Placeholder"] = ""
	tmpl["Description"] = ""
	tmpl["Required"] = ""
	if required {
		tmpl["Required"] = "required"
	}
	if required {
		tmpl["Class"] = "hint hint-req"
	}
	tmpl["Type"] = "text"
	tmpl["Multiple"] = ""
	tmpl["Selected"] = []string{}
	schemaRecordMap := *smap
	if strings.Contains(skey, ".") {
		// we passed structKey with subkey use subkey for look-up in a map
		arr := strings.Split(skey, ".")
		skey = arr[1] // use subkey
	}
	if r, ok := schemaRecordMap[skey]; ok {
		if r.Type == "list_struct" || r.Type == "struct" {
			// we don't need to build web UI element for struct type as it will be handled differently
			// via formStructEntry function
			return ""
		}
		if r.Section == section {
			if r.Type == "list_str" || r.Type == "list" {
				tmpl["List"] = true
				switch values := r.Value.(type) {
				case []any:
					var vals, selected []string
					if defaultValue != "" {
						selected = append(selected, defaultValue)
					}
					tmpl["Selected"] = selected
					for _, v := range values {
						if v != defaultValue && v != "" {
							strVal := fmt.Sprintf("%v", v)
							if !utils.InList(strVal, selected) {
								vals = append(vals, strVal)
							}
						}
					}
					vals = utils.List2Set(vals)
					// for list data types, e.g. array of strings
					// we can clearly define empty default value. And if attribute is not required
					// we should use empty value first
					if !required && defaultValue == "" {
						vals = append([]string{""}, vals...)
					}
					tmpl["Value"] = vals
				default:
					tmpl["Value"] = []string{}
				}
			} else if r.Type == "bool" || r.Type == "boolean" {
				tmpl["List"] = true
				if r.Value == true {
					tmpl["Value"] = []string{"", "true", "false"}
				} else {
					tmpl["Value"] = []string{"", "false", "true"}
				}
				if defaultValue != "" {
					if defaultValue == "true" {
						tmpl["Value"] = []string{"true", "false"}
					} else {
						tmpl["Value"] = []string{"false", "true"}
					}
				}
			} else {
				if r.Value != nil {
					switch values := r.Value.(type) {
					case []any:
						tmpl["List"] = true
						var vals []string
						for _, v := range values {
							strVal := fmt.Sprintf("%v", v)
							vals = append(vals, strVal)
						}
						vals = utils.List2Set(vals)
						// for non list data types, e.g. array of floats
						// we cannot clearly define empty default value as it should come from
						// schema itself, and tehrefire we do not update vals with first empty value
						tmpl["Value"] = vals
					default:
						tmpl["Value"] = fmt.Sprintf("%v", r.Value)
					}
				}
			}
			if r.Multiple {
				tmpl["Multiple"] = "multiple"
			}
			desc := r.Description
			if desc == "" {
				desc = "Not Available"
			}
			tmpl["Description"] = desc
			tmpl["Placeholder"] = r.Placeholder
		}
	}
	return server.TmplPage(StaticFs, "form_entry.tmpl", tmpl)
}

// helper function to build struct form entry
func formStructEntry(schemaFileName, structKey, subKey string) string {
	schema, err := _smgr.Load(schemaFileName)
	if err != nil {
		msg := fmt.Sprintf("unable to load %s error %v", schemaFileName, err)
		log.Println("ERROR: ", msg)
		return msg
	}
	var section string
	// structKey represents <struct_entry.sub_key>
	arr := strings.Split(structKey, ".")
	if len(arr) > 0 {
		// we use struct entry as section for web UI
		section = arr[0]
	}
	smap := schema.Map
	var subRecords []string
	for key, srec := range smap {
		if key == subKey {
			if srec.Section == "" {
				// update schema record entry with proper section if is empty
				srec.Section = section
				smap[key] = srec
			}
			// here we pass structKey to formEntry to ensure that HTML input name value will use
			// see formEntry logic how it handles given key with period symbol in it
			rec := formEntry(&smap, structKey, section, !srec.Optional, nil)
			subRecords = append(subRecords, rec)
		}
	}
	return strings.Join(subRecords, "\n<br/>\n")
}

// helper function to parser form values
func parseValue(schema *beamlines.Schema, key string, items []string) (any, error) {
	r, ok := schema.Map[key]
	if !ok {
		if srvConfig.Config.Frontend.TestMode && utils.InList(key, srvConfig.Config.CHESSMetaData.SkipKeys) {
			return "", nil
		}
		msg := fmt.Sprintf("No key %s found in schema map", key)
		log.Printf("ERROR: %s", msg)
		return false, errors.New(msg)
	} else if r.Type == "list_str" {
		switch r.Value.(type) {
		case []any:
			return items, nil
		case []string:
			return items, nil
		default:
			nonEmptyItems := []string{}
			for _, v := range items {
				if v != "" {
					nonEmptyItems = append(nonEmptyItems, v)
				}
			}
			return nonEmptyItems, nil
		}
		return items, nil
	} else if strings.HasPrefix(r.Type, "list_int") {
		// parse given values to int data type
		var vals []int
		for _, values := range items {
			for _, val := range strings.Split(values, " ") {
				v, err := strconv.Atoi(val)
				if err == nil {
					vals = append(vals, v)
				} else {
					msg := fmt.Sprintf("ERROR: unable to parse input '%v' into int data-type, %v", items, err)
					return items, errors.New(msg)
				}
			}
		}
		return vals, nil
	} else if strings.HasPrefix(r.Type, "list_float") {
		// parse given values to float data type
		var vals []float64
		for _, values := range items {
			for _, val := range strings.Split(values, " ") {
				// if passed web form value contained comma we will replace it
				val = strings.ReplaceAll(val, ",", "")
				val = strings.Trim(val, " ")
				v, err := strconv.ParseFloat(val, 64)
				if err == nil {
					vals = append(vals, v)
				} else {
					msg := fmt.Sprintf("ERROR: unable to parse input '%v' into float data-type, %v", items, err)
					return items, errors.New(msg)
				}
			}
		}
		return vals, nil
	} else if r.Type == "string" {
		val := strings.Trim(strings.Join(items, ""), "")
		return val, nil
	} else if r.Type == "bool" {
		v, err := strconv.ParseBool(items[0])
		if err == nil {
			return v, nil
		}
		msg := fmt.Sprintf("Unable to parse boolean value for key=%s, please come back to web form and choose either true or false", key)
		log.Printf("ERROR: %s", msg)
		return false, errors.New(msg)
	} else if strings.HasPrefix(r.Type, "int") {
		v, err := strconv.ParseInt(items[0], 10, 64)
		if err == nil {
			if r.Type == "int64" {
				return int64(v), nil
			} else if r.Type == "int32" {
				return int32(v), nil
			} else if r.Type == "int16" {
				return int16(v), nil
			} else if r.Type == "int8" {
				return int8(v), nil
			} else if r.Type == "int" {
				return int(v), nil
			}
			return v, nil
		}
		return 0, fmt.Errorf("[Frontend.main.parseValue] strconv.ParseInt error: %w", err)
	} else if strings.HasPrefix(r.Type, "float") {
		v, err := strconv.ParseFloat(items[0], 64)
		if err == nil {
			if r.Type == "float32" {
				return float32(v), nil
			}
			return v, nil
		}
		return 0.0, fmt.Errorf("[Frontend.main.parseValue] strconv.ParseFloat error: %w", err)
	}
	msg := fmt.Sprintf("Unable to parse form value for key %s", key)
	log.Printf("ERROR: %s", msg)
	return 0, errors.New(msg)
}

// helper function to retrieve files from web user record form
func formFiles(val any) []string {
	var out []string
	switch files := val.(type) {
	case []string:
		for _, f := range files {
			if strings.Contains(f, ",") {
				for _, v := range strings.Split(f, ",") {
					v = strings.Replace(v, "\n", "", -1)
					v = strings.Replace(v, "\r", "", -1)
					v = strings.Trim(v, " ")
					if v != "" {
						out = append(out, v)
					}
				}
			} else if strings.Contains(f, "\r") {
				for _, v := range strings.Split(f, "\r") {
					v = strings.Replace(v, "\n", "", -1)
					v = strings.Replace(v, "\r", "", -1)
					v = strings.Trim(v, " ")
					if v != "" {
						out = append(out, v)
					}
				}
			} else if strings.Contains(f, "\n") {
				for _, v := range strings.Split(f, "\n") {
					v = strings.Replace(v, "\n", "", -1)
					v = strings.Replace(v, "\r", "", -1)
					v = strings.Trim(v, " ")
					if v != "" {
						out = append(out, v)
					}
				}
			} else {
				if f != "" {
					out = append(out, f)
				}
			}
		}
	}
	return out
}

// helper function to extract beamline, btr, cycle, sample_name parts from did
func extractParts(did string) (string, string, string, string) {
	var beamline, btr, cycle, sample_name string
	for _, part := range strings.Split(did, "/") {
		if strings.HasPrefix(part, "beamline=") {
			beamline = strings.Replace(part, "beamline=", "", -1)
		}
		if strings.HasPrefix(part, "btr=") {
			btr = strings.Replace(part, "btr=", "", -1)
		}
		if strings.HasPrefix(part, "cycle=") {
			cycle = strings.Replace(part, "cycle=", "", -1)
		}
		if strings.HasPrefix(part, "sample_name=") {
			sample_name = strings.Replace(part, "sample_name=", "", -1)
		}
	}
	return beamline, btr, cycle, sample_name
}

// helper function to make provenance links from a links of given dids
func makeProvenanceLinks(dids []string) []string {
	var out []string
	tmpl := server.MakeTmpl(StaticFs, "Provenance information")
	for _, did := range dids {
		if did == "" {
			continue
		}
		tmpl["did"] = did
		page := server.TmplPage(StaticFs, "did_links.tmpl", tmpl)
		out = append(out, page)
	}
	return out
}

// helper function to update UserMetaData
func updateUserMetaData(did string, val any) error {
	var rec map[string]any
	var mrec map[string]any
	switch v := val.(type) {
	case map[string]any:
		rec = v
	default:
		msg := fmt.Sprintf("Unsupport data-type %T for user medata record", val)
		return errors.New(msg)
	}
	if vvv, ok := rec["metadata"]; ok {
		switch v := vvv.(type) {
		case map[string]any:
			mrec = v
		default:
			msg := fmt.Sprintf("Unsupport data-type %T for user medata record", val)
			return errors.New(msg)
		}
	} else {
		mrec = rec
	}
	mrec["did"] = did
	rurl := fmt.Sprintf("%s/record", srvConfig.Config.Services.UserMetaDataURL)
	data, err := json.Marshal(mrec)
	if err != nil {
		return fmt.Errorf("[Frontend.main.updateUserMetaData] json.Marshal error: %w", err)
	}
	_httpWriteRequest.GetToken()
	resp, err := _httpWriteRequest.Post(rurl, "application/json", bytes.NewBuffer(data))
	if err != nil {
		return fmt.Errorf("[Frontend.main.updateUserMetaData] _httpWriteRequest.Post error: %w", err)
	}
	defer resp.Body.Close()
	data, err = io.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("[Frontend.main.updateUserMetaData] io.ReadAll error: %w", err)
	}

	var sresp services.ServiceResponse
	err = json.Unmarshal(data, &sresp)
	if err != nil {
		return fmt.Errorf("[Frontend.main.updateUserMetaData] json.Unmarshal error: %w", err)
	}
	if sresp.SrvCode != 0 || sresp.HttpCode != http.StatusOK {
		return errors.New(sresp.String())
	}
	return nil
}

func convertTypes(subschema *beamlines.Schema, records []map[string]string) []map[string]any {
	out := make([]map[string]any, 0, len(records))
	smap := subschema.Map

	for _, r := range records {
		row := make(map[string]any)

		for k, v := range r {
			if t, ok := smap[k]; ok {
				if value, err := utils.Convert2dtype(v, t.Type); err == nil {
					row[k] = value
					log.Printf("converted %s=%q (%T) -> %v (%T)", k, v, v, value, value)
					continue
				}
				// conversion failed → fall back to string
				log.Printf("WARNING: failed to convert %s=%q to %s, keeping string", k, v, t.Type)
			}

			// unknown field or failed conversion
			row[k] = v
		}

		out = append(out, row)
	}

	return out
}

// ELogEntry defines Elog entry structure
type ELogEntry struct {
	Did      string    `json:"did,omitempty"`
	Text     string    `json:"text"`
	User     string    `json:"user"`
	DidHash  string    `json:"didhash,omitempty"`
	ImageURL string    `json:"image_url,omitempty"`
	Date     time.Time `json:"date"`
}

// helper function to fetch elog entries
func fetchELogEntries(did, user string) []ELogEntry {
	var records []map[string]any
	var entries []ELogEntry
	_httpReadRequest.GetToken()
	rurl := fmt.Sprintf("%s/search?did=%s", srvConfig.Config.ELogServiceURL, did)
	resp, err := _httpReadRequest.Get(rurl)
	if err != nil {
		log.Println("WARNING: unable to get datahub didHashes, error:", err)
		return entries
	}
	defer resp.Body.Close()
	data, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Println("WARNING: unable to get elog entries, error:", err)
		return entries
	}
	err = json.Unmarshal(data, &records)
	if err != nil {
		log.Println("WARNING: unable to get elog entries, error:", err)
	}
	for _, r := range records {
		entry := ELogEntry{}
		if val, ok := r["text"]; ok {
			entry.Text = fmt.Sprintf("%v", val)
		}
		if val, ok := r["user"]; ok {
			entry.User = fmt.Sprintf("%v", val)
		}
		if val, ok := r["image_url"]; ok {
			entry.ImageURL = fmt.Sprintf("%v", val)
		}
		if val, ok := r["date"]; ok {
			if ts, err := getUnixNano(val); err == nil {
				entry.Date = time.Unix(0, ts).UTC()
			}
		}
		entries = append(entries, entry)
	}
	return entries
}

// helper function to convert map value to unix nano
func getUnixNano(v any) (int64, error) {
	switch t := v.(type) {
	case int64:
		return t, nil
	case int:
		return int64(t), nil
	case float64:
		return int64(t), nil
	case string:
		ts, err := strconv.ParseInt(t, 10, 64)
		if err != nil {
			return 0, fmt.Errorf("invalid date string: %v", err)
		}
		return ts, nil
	default:
		return 0, fmt.Errorf("unsupported type for date: %T", v)
	}
}

// helper function to get total number of records from upstream service
func countMetadataRecords() (int, error) {
	// get total number of metadata records in FOXDEN
	_httpReadRequest.GetToken()
	rec := services.ServiceRequest{
		Client:       "frontend",
		ServiceQuery: services.ServiceQuery{Query: "{}", Idx: 0, Limit: 1},
	}
	data, err := json.Marshal(rec)
	if err != nil {
		return 0, err
	}
	rurl := fmt.Sprintf("%s/count", srvConfig.Config.Services.MetaDataURL)
	resp, err := _httpReadRequest.Post(rurl, "application/json", bytes.NewBuffer(data))
	if err != nil {
		return 0, err
	}
	body, err := io.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return 0, err
	}
	var nrec int
	if err = json.Unmarshal(body, &nrec); err != nil {
		return 0, err
	}
	return nrec, nil
}

// NoteEntry defines Elog entry structure
type NoteEntry struct {
	Did  string `json:"did"`
	Text string `json:"text"`
	User string `json:"user"`
	Date uint64 `json:"date"`
}

func getNotes(dids []string) map[string]string {
	rurl := fmt.Sprintf("%s/records", srvConfig.Config.Services.ELogServiceURL)
	data, err := json.Marshal(dids)
	if err != nil {
		log.Printf("ERROR: unable to marshal dids, error: %v", err)
		return nil
	}
	_httpReadRequest.GetToken()
	resp, err := _httpReadRequest.Post(rurl, "application/json", bytes.NewBuffer(data))
	if err != nil {
		log.Printf("ERROR: unable to request notes from ELogService, error: %v", err)
		return nil
	}
	body, err := io.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		log.Printf("ERROR: unable to read log entries ELogService, error: %v", err)
		return nil
	}
	var out []NoteEntry
	err = json.Unmarshal(body, &out)
	if err != nil {
		log.Printf("ERROR: unable to unmarshal http response body, error: %v", err)
	}
	return notesToMap(out)
}

// noteToMap converts given note entries to a map used in UI
func notesToMap(notes []NoteEntry) map[string]string {
	// keep latest note per Did
	latest := make(map[string]NoteEntry)

	for _, n := range notes {
		if existing, ok := latest[n.Did]; !ok || n.Date > existing.Date {
			latest[n.Did] = n
		}
	}

	// build final map[string]string
	result := make(map[string]string, len(latest))

	for did, n := range latest {
		t := time.Unix(0, int64(n.Date))
		result[did] = fmt.Sprintf(
			"%s @ %s",
			n.User,
			t.Format(time.DateTime),
		)
	}

	return result
}

func addAffiliation(members []string) []string {
	var out []string
	for _, m := range members {
		// uid suffix is added by golib/services/chess.go in BtrMembersUids
		arr := strings.Split(m, " uid ")
		if len(arr) != 2 {
			log.Println("WARNING: unable to extract member's uid:", m)
			continue
		}
		name := arr[0]
		uid := arr[1]
		_httpReadRequest.GetToken()
		rurl := fmt.Sprintf("%s/affiliations?uid=%s", srvConfig.Config.BeamPassURL, uid)
		resp, err := _httpReadRequest.Get(rurl)
		if err != nil {
			log.Println("WARNING: unable to get user affiliation, error:", err)
			out = append(out, name)
			continue
		}
		defer resp.Body.Close()
		data, err := io.ReadAll(resp.Body)
		if err != nil {
			log.Println("WARNING: unable to read response body, error:", err)
			out = append(out, name)
			return members
		}
		// deserialize data into struct and construct new string name
		var records []AffiliationInfo
		if err := json.Unmarshal(data, &records); err == nil {
			for _, r := range records {
				if r.Affiliation != "" {
					out = append(out, r.Affiliation)
					break
				}
			}
		}
	}
	return out
}

type AffiliationInfo struct {
	Affiliation string `json:"affiliation"`
}