$\underline{\space{\bf\text{0.}}\space}$ Start by:
$$
\begin{align}
I =& -6\int\limits_{0}^{\pi/6} x^2 \log^2\left(2\sin(x)\right) \,dx
\end{align}
$$
$\underline{\space{\bf\text{1.}}\space}$ Change to $ \displaystyle \cos(x)=\sin\left(\frac{\pi}{2}-x\right) $ in order to use the identity:
$$
\begin{align}
&2\cos(x) = e^{{\small+}ix} + e^{{\small-}ix} \implies \\[1mm]
&\log\left(1+e^{{\small-}i\,2x}\right) = \log\left(2\cos(x)\right) + ix \\[1mm]
&\log^2\left(1+e^{{\small-}i\,2x}\right) = \log^2\left(2\cos(x)\right) - x^2 + i\,2x\log\left(2\cos(x)\right) \\
\end{align}
$$
And because of the integration range, consider only the REAL part:
$$
\begin{align}
I =& -6\int\limits_{\pi/3}^{\pi/2} \left(\frac{\pi}{2}-x\right)^2 \log^2\left(2\cos(x)\right) \,dx \\[1mm]
=& -6\,\,\Re{\int\limits_{\pi/3}^{\pi/2}} \left(\frac{\pi}{2}-x\right)^2 \left[x^2+\log^2\left(1+e^{{\small-}i\,2x}\right)\right] dx \\[1mm]
=& \color{blue}{-6\int\limits_{\pi/3}^{\pi/2} \left(\frac{\pi}{2}-x\right)^2 x^2 \,dx} -6\,\,\Re{\int\limits_{\pi/3}^{\pi/2}} \left(\frac{\pi}{2}-x\right)^2 \log^2\left(1+e^{{\small-}i\,2x}\right) \,dx \\[1mm]
=& \color{blue}{-\frac{17\,\pi^5}{12960}} -6\,\,\Re{\int\limits_{\pi/3}^{\pi/2}} \left(\frac{\pi}{2}-x\right)^2 \log^2\left(1+e^{{\small-}i\,2x}\right) \,dx \\[1mm] \end{align}
$$
$\underline{\space{\bf\text{2.}}\space}$ Substitute: $ \displaystyle -e^{{\small-}i\,2x} \mapsto x \implies x \mapsto \frac{i}{2}\log(-x) $
Also use: $ \displaystyle \log(-x)=\log(x)-i\pi\, $
And let: $ \displaystyle \color{red}{a} =-e^{{\small-}i\,2\pi/3} =\frac{1}{2}+i\,\frac{\sqrt{3}}{2} $
$$
\begin{align}
I =& -\frac{17\,\pi^5}{12960} -6\,\,\Re{\int\limits_{a}^{1}} \left[\frac{\pi}{2}-\frac{i}{2}\left(\log(x)-i\pi\right)\right]^2 \log^2(1-x) \,\frac{i}{2x} \,dx \\[1mm]
=& -\frac{17\,\pi^5}{12960} \color{red}{+}\frac{3}{4}\,\,\color{red}{\Re}{\int\limits_{a}^{1}} \,\color{red}{i}\,\frac{\log^2(x)\log^2(1-x)}{x} \,dx \\[1mm]
=& -\frac{17\,\pi^5}{12960} \color{red}{-}\frac{3}{4}\,\,\color{red}{\Im}{\int\limits_{a}^{1}} \,\frac{\log^2(x)\log^2(1-x)}{x} \,dx
\end{align}
$$
$\color{red}{\underline{\space{\bf\text{3.}}\space\text{(the main trick)}\colon}}$ The beauty of the last integral is the switching between $\,(x)\,$ and $\,(1-x)\,$:
$$
\begin{align}
\color{blue}{+\int\,\frac{\log^2(x)\log^2(1-x)}{x}\,dx} = \color{blue}{-\int\,\frac{\log^2(1-x)\log^2(x)}{1-x}\,dx}
\end{align}
$$
Hence, using this property and substituting $\, \displaystyle x \mapsto 1/2+x \,$,
considering the integral range and the IMAGINARY part:
$$
\begin{align}
\int\limits_{a}^{1}\,\frac{\log^2(x)\log^2(1-x)}{x}\,dx =& \int\limits_{i\sqrt{3}/2}^{1/2}\,\frac{\log^2(1/2+x)\log^2(1/2-x)}{1/2+x}\,dx \\[1mm]
\int\limits_{a}^{1}\,\frac{\log^2(x)\log^2(1-x)}{1-x}\,dx =& \int\limits_{i\sqrt{3}/2}^{1/2}\,\frac{\log^2(1/2+x)\log^2(1/2-x)}{1/2-x}\,dx
\end{align}
$$
Thus,
$$
\begin{align}
I =& -\frac{17\,\pi^5}{12960} -\frac{3}{4}\,\,\Im{\int\limits_{i\sqrt{3}/2}^{1/2}} \,{\log^2(1/2-x)\log^2(1/2+x)} \left[\frac{1/2}{1/2-x}+\frac{1/2}{1/2+x}\right] dx \\[2mm]
=& -\frac{17\,\pi^5}{12960} -\frac{3}{8}\,\,\Im{\int\limits_{i\sqrt{3}/2}^{1/2}} \,\frac{\log^2(1/2-x)}{1/2-x} \,\frac{\log^2(1/2+x)}{1/2+x} \,dx \quad\color{red}{\left\{\small{x}\mapsto{x/2}\right\}} \\[2mm]
=& \color{blue}{-\frac{17\,\pi^5}{12960} -\frac{3}{4}\,\,\Im{\int\limits_{i\sqrt{3}}^{1}} \,\frac{\log^2\left((1-x)/2\right)}{1-x} \,\frac{\log^2\left((1+x)/2\right)}{1+x} \,dx}
\end{align}
$$
$\underline{\space{\bf\text{4.}}\space}$ Similar to this question, use the identity:
$$
\begin{align}
12\log^2(x)\,\log^2(y) =& \,\left[\log(x)+\log(y)\right]^4 + \left[\log(x)-\log(y)\right]^4 -2\left[\log(x)\right]^4 -2\left[\log(y)\right]^4 \\[1mm]
=& \,\log^4(x\,y) + \log^4(x/y) -2\log^4(x) -2\log^4(y)
\end{align}
$$
To split the last integral into four integrals:
$$
\begin{align}
\color{brown}{I_{_1}(x)} =&\, \color{brown}{\int\frac{\log^4\left((1-x)\,(1+x)\right)}{1-x^2}\,dx} \\[1mm]
\color{magenta}{I_{_2}(x)} =&\, \color{magenta}{\int\frac{\log^4\left((1-x)/(1+x)\right)}{1-x^2}\,dx} \\[1mm]
\color{olive}{I_{_3}(x)} =&\, \color{olive}{\int\frac{\log^4\left(1-x\right)}{1-x^2}\,dx} \\[1mm]
\color{orange}{I_{_4}(x)} =&\, \color{orange}{\int\frac{\log^4\left(1+x\right)}{1-x^2}\,dx}
\end{align}
$$
And with some WA help (specially for $I_{_1}$):
$$
\begin{align}
\color{brown}{I_{_1}(x)}
=&\, \color{brown}{ \frac{1}{x}\sqrt{\frac{x^2}{1-x^2}}\,\times } \\
&\color{brown}{ \,\,[\,\,384\,\log^0\left({\small({1-x^2})/{4}}\right)\, {_{\small6}F_{\small5}}{\left({\small\frac12,\frac12,\frac12,\frac12,\frac12,\frac12};\,{\small\frac32,\frac32,\frac32,\frac32,\frac32};\,{\small\frac{1}{1-x^2}}\right)} } \\
& \color{brown}{ +\,192\,\log^1\left({\small({1-x^2})/{4}}\right)\, {_{\small5}F_{\small4}}{\left({\small\frac12,\frac12,\frac12,\frac12,\frac12};\,{\small\frac32,\frac32,\frac32,\frac32};\,{\small\frac{1}{1-x^2}}\right)} } \\
& \color{brown}{ +\,\,\,48\,\,\log^2\left({\small({1-x^2})/{4}}\right)\, {_{\small4}F_{\small3}}{\left({\small\frac12,\frac12,\frac12,\frac12};\,{\small\frac32,\frac32,\frac32};\,{\small\frac{1}{1-x^2}}\right)} } \\
& \color{brown}{ +\,\,\,\,8\,\,\,\,\log^3\left({\small({1-x^2})/{4}}\right)\, {_{\small3}F_{\small2}}{\left({\small\frac12,\frac12,\frac12};\,{\small\frac32,\frac32};\,{\small\frac{1}{1-x^2}}\right)} } \\
& \color{brown}{ +\,\,\,\,1\,\,\,\,\log^4\left({\small({1-x^2})/{4}}\right)\, {_{\small2}F_{\small1}}{\left({\small\frac12,\frac12};\,{\small\frac32};\,{\small\frac{1}{1-x^2}}\right)}] } \\[3mm]
\color{magenta}{I_{_2}(x)} =&\, \color{magenta}{ -{\small\frac{1}{10}}\log^5\left({\small\frac{1-x}{1+x}}\right) } \\[3mm]
\color{olive}{I_{_3}(x)} =&\, \color{olive}{ -{\small\frac{1}{10}}\log^5\left({\small\frac{1-x}{2}}\right) } \\
&\, \color{olive}{ -\,{\small\frac{1}{2}}\,\log^4\left({\small\frac{1-x}{2}}\right) \text{ Li}_{\small1}{\left({\small\frac{1-x}{2}}\right)} } \\
&\, \color{olive}{ +\,\,2\,\,\log^3\left({\small\frac{1-x}{2}}\right) \text{ Li}_{\small2}{\left({\small\frac{1-x}{2}}\right)} } \\
&\, \color{olive}{ -\,\,6\,\,\log^2\left({\small\frac{1-x}{2}}\right) \text{ Li}_{\small3}{\left({\small\frac{1-x}{2}}\right)} } \\
&\, \color{olive}{ +\,12\,\log^1\left({\small\frac{1-x}{2}}\right) \text{ Li}_{\small4}{\left({\small\frac{1-x}{2}}\right)} } \\
&\, \color{olive}{ -\,12\,\log^0\left({\small\frac{1-x}{2}}\right) \text{ Li}_{\small5}{\left({\small\frac{1-x}{2}}\right)} } \\[3mm]
\color{orange}{I_{_4}(x)} =&\, \color{orange}{ +{\small\frac{1}{10}}\log^5\left({\small\frac{1+x}{2}}\right) } \\
&\, \color{orange}{ +\,{\small\frac{1}{2}}\,\log^4\left({\small\frac{1+x}{2}}\right) \text{ Li}_{\small1}{\left({\small\frac{1+x}{2}}\right)} } \\
&\, \color{orange}{ -\,\,2\,\,\log^3\left({\small\frac{1+x}{2}}\right) \text{ Li}_{\small2}{\left({\small\frac{1+x}{2}}\right)} } \\
&\, \color{orange}{ +\,\,6\,\,\log^2\left({\small\frac{1+x}{2}}\right) \text{ Li}_{\small3}{\left({\small\frac{1+x}{2}}\right)} } \\
&\, \color{orange}{ -\,12\,\log^1\left({\small\frac{1+x}{2}}\right) \text{ Li}_{\small4}{\left({\small\frac{1+x}{2}}\right)} } \\
&\, \color{orange}{ +\,12\,\log^0\left({\small\frac{1+x}{2}}\right) \text{ Li}_{\small5}{\left({\small\frac{1+x}{2}}\right)} }
\end{align}
$$
$\underline{\space{\bf\text{5.}}\space}$ Define: $ \displaystyle f(x) = \color{brown}{I_{_1}{(x)}} + \color{magenta}{I_{_2}{(x)}} - \color{olive}{2\,I_{_3}{(x)}} - \color{orange}{2\,I_{_4}{(x)}} $
And calculate the limits at the integral boundaries:
$$
\begin{align}
\lim_{x\to 1}\,f(x)\, =&\, -4\,\pi^2\zeta(3)+48\,\zeta(5)-i\,\frac{19\,\pi^5}{30} \\[1mm]
\lim_{x\to i\sqrt{3}}\,f(x)\, =&\, +i\,\frac{\pi^5}{81}-i\,192 \,{_{\small6}F_{\small5}}{\left({\small\frac12,\frac12,\frac12,\frac12,\frac12,\frac12};\,{\small\frac32,\frac32,\frac32,\frac32,\frac32};\,{\small\frac14}\right)}
\end{align}
$$
$\underline{\space{\bf\text{6.}}\space}$ Simplify:
$$
\begin{align}
&\quad\,\, I = -\frac{17\,\pi^5}{12960} -\frac{3}{4}\,\frac{1}{12}\,\,\Im\left[\,f\left(1\right)-f\left(i\sqrt{3}\right)\,\right] \\[1mm]
&\boxed{\quad \color{red}{I =\frac{253\,\pi^5}{6480} - 12\,{_{\small6}F_{\small5}}{\left({\small\frac12,\frac12,\frac12,\frac12,\frac12,\frac12};\,{\small\frac32,\frac32,\frac32,\frac32,\frac32};\,{\small\frac14}\right)}} \quad \\}
\end{align}
$$